2021-05-07T19:00:34.378 INFO:root:teuthology version: 1.1.0-19220a3b 2021-05-07T19:00:34.380 DEBUG:teuthology.run:Teuthology command: teuthology --name sage-2021-05-07_18:25:10-upgrade:pacific-x:stress-split-wip-sage3-testing-2021-05-07-1141-distro-basic-smithi --owner scheduled_sage@teuthology --archive /home/teuthworker/archive/sage-2021-05-07_18:25:10-upgrade:pacific-x:stress-split-wip-sage3-testing-2021-05-07-1141-distro-basic-smithi/6104808 --verbose --description upgrade:pacific-x:stress-split/{0-distro/centos_8.2_kubic_stable 0-roles 1-start 2-first-half-tasks/readwrite 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} -- /home/teuthworker/archive/sage-2021-05-07_18:25:10-upgrade:pacific-x:stress-split-wip-sage3-testing-2021-05-07-1141-distro-basic-smithi/6104808/orig.config.yaml 2021-05-07T19:00:34.453 DEBUG:teuthology.report:Pushing job info to http://paddles.front.sepia.ceph.com/ 2021-05-07T19:00:34.517 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/sage-2021-05-07_18:25:10-upgrade:pacific-x:stress-split-wip-sage3-testing-2021-05-07-1141-distro-basic-smithi/6104808 branch: wip-sage3-testing-2021-05-07-1141 description: upgrade:pacific-x:stress-split/{0-distro/centos_8.2_kubic_stable 0-roles 1-start 2-first-half-tasks/readwrite 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} email: null first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' first_in_suite: false job_id: '6104808' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: sage-2021-05-07_18:25:10-upgrade:pacific-x:stress-split-wip-sage3-testing-2021-05-07-1141-distro-basic-smithi nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: '8.2' overrides: admin_socket: branch: wip-sage3-testing-2021-05-07-1141 ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) log-whitelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: 60a03ee87c86b2c41aae70c5787a604c7a05af35 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: osd default pool size: 2 install: ceph: sha1: 60a03ee87c86b2c41aae70c5787a604c7a05af35 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: qa-pacific-x-stress sha1: 52e70315b88a579fda581af21fc4e8f3d53bc224 owner: scheduled_sage@teuthology priority: 75 repo: git://git.ceph.com/ceph-ci.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' sha1: 60a03ee87c86b2c41aae70c5787a604c7a05af35 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: octopus clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: octopus clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: octopus clients: client.0: - rbd/test_librbd.sh - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' suite: upgrade:pacific-x:stress-split suite_branch: qa-pacific-x-stress suite_path: /home/teuthworker/src/github.com_liewegas_ceph_52e70315b88a579fda581af21fc4e8f3d53bc224/qa suite_relpath: qa suite_repo: https://github.com/liewegas/ceph suite_sha1: 52e70315b88a579fda581af21fc4e8f3d53bc224 targets: smithi058.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDvlfNfaLCzBcdFOZ2eKApGa/1VIPKa0EOtGxW+1xGn2fSPqvbCk9XuwmWgYxCVelh7ptJ9qlEEvrJVnNRG/AxJkBufvQfeSbS8T4LCRpBgw322NzePgrD5z5sGzUkGh1AWHzo3ySf9y3GcvbtkUEtOTMOdaNhtO8nPfUfRV1vzmMukoU1cFyVwf3kNzYoxNgJAcv7Z6IleZSdJ/K+hL1o3zYJ5ExNwtfwcpndSknLbHNZyU0T9nHC4IFUP4ZRnLLMxnPvMd7liDbvKnuDr1wCbLouHSTWlDryD2QMr1nPyDd25fM0cVLxh/MbYwy17MOh4HV1TeWZ6Aw7UXSCkLlsnUUltRqWE8/eh8YrrrrTxpvsPUC2dNuzmvwaGg9pyJD+dDHCEFMHoH7IleyGRXdGD6+vOavWABP9fE6UFmJ0ENS4Rx9698dkkaPZb7kjuNg6VltksL/LdCewEOBbqeP2hREp4IaL0a+LBxIdx9N691KBOM2MD9cbJV2ncvDcBOrc= smithi125.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDtm/CF2qYndmPdsKpa7fZE+/t9ilUYP2gklEQb2PvjLt0uTAiU0xbMg+U36UD233f4UmT3MJxYubAOFKaM3mpUAGTSH9DtbKpAhp+7s6SQBnZZ6/1v8TImHrsApiicbyuUgnJiQnOtim5IJQUc+EGxORmVgIBwxkKh8j/NkY+i5xmGS677iTiohU64dAyMxyktY2lTmrbNh1T2jq64apMXH1d5xtf3xnfs4iWDQtT7DmkLpaPtfRSA6Wyjk7JBTOVb0PSQ9/is41EobOla+RuNb+eMKRp2EReYA5Y0ADIwg3KfSov1Fw7yZHQEKCmpntJgstr54+fDELytKZtWSH5Esy37+XBVJbhg8N6f4bCUz9dQq46r4EGzhK+u2Qg+yMl3oOVrbPG6NWSTiUh3uc+6w4bYyrv1FRSwn18CXHbuEQILlHFPVESHK+bLrfMEUaSlxjlXXZBTWTmWdtCxkSdlYCyyj16TbZ4G6t7Wt783iGt9m3n/tfOpvsU4QmXna+s= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module disable container-tools - sudo dnf -y install 'dnf-command(copr)' - sudo dnf -y copr enable rhcontainerbot/container-selinux - sudo curl -L -o /etc/yum.repos.d/devel:kubic:libcontainers:stable.repo https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/CentOS_8/devel:kubic:libcontainers:stable.repo - sudo dnf remove -y podman - sudo dnf -y install podman - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - install: branch: pacific - cephadm: add_mons_via_daemon_add: true avoid_pacific_features: true cephadm_branch: pacific cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: docker.io/ceph/daemon-base:latest-pacific - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology_branch: master teuthology_sha1: 19220a3bd6e252c6e8260827019668a766d85490 timestamp: 2021-05-07_18:25:10 tube: smithi user: sage verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.26341 2021-05-07T19:00:34.517 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_liewegas_ceph_52e70315b88a579fda581af21fc4e8f3d53bc224/qa; will attempt to use it 2021-05-07T19:00:34.518 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_liewegas_ceph_52e70315b88a579fda581af21fc4e8f3d53bc224/qa/tasks 2021-05-07T19:00:34.518 INFO:teuthology.run_tasks:Running task internal.check_packages... 2021-05-07T19:00:34.520 INFO:teuthology.task.internal:Checking packages... 2021-05-07T19:00:34.540 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'basic' and ceph hash '60a03ee87c86b2c41aae70c5787a604c7a05af35' 2021-05-07T19:00:34.540 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2021-05-07T19:00:34.541 INFO:teuthology.packaging:ref: None 2021-05-07T19:00:34.541 INFO:teuthology.packaging:tag: None 2021-05-07T19:00:34.541 INFO:teuthology.packaging:branch: wip-sage3-testing-2021-05-07-1141 2021-05-07T19:00:34.542 INFO:teuthology.packaging:sha1: 60a03ee87c86b2c41aae70c5787a604c7a05af35 2021-05-07T19:00:34.542 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=wip-sage3-testing-2021-05-07-1141 2021-05-07T19:00:34.762 INFO:teuthology.task.internal:Found packages for ceph version 17.0.0-3987.g60a03ee8 2021-05-07T19:00:34.763 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2021-05-07T19:00:34.773 INFO:teuthology.task.internal:no buildpackages task found 2021-05-07T19:00:34.774 INFO:teuthology.run_tasks:Running task internal.save_config... 2021-05-07T19:00:34.799 INFO:teuthology.task.internal:Saving configuration 2021-05-07T19:00:34.824 INFO:teuthology.run_tasks:Running task internal.check_lock... 2021-05-07T19:00:34.834 INFO:teuthology.task.internal.check_lock:Checking locks... 2021-05-07T19:00:34.853 DEBUG:teuthology.task.internal.check_lock:machine status is {'is_vm': False, 'locked': True, 'locked_since': '2021-05-07 18:55:31.202141', 'description': '/home/teuthworker/archive/sage-2021-05-07_18:25:10-upgrade:pacific-x:stress-split-wip-sage3-testing-2021-05-07-1141-distro-basic-smithi/6104808', 'locked_by': 'scheduled_sage@teuthology', 'up': True, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAn61VoeUADBBslIPwlpfrso2T9QFQBdpQRY9F6iG7kafZ4u0/sAP7cw++S7Svgb/T1wTcZVIdzpZE+fPjdxYzw=', 'os_version': '8', 'machine_type': 'smithi', 'vm_host': None, 'os_type': 'centos', 'arch': 'x86_64', 'mac_address': None, 'name': 'smithi058.front.sepia.ceph.com'} 2021-05-07T19:00:34.868 DEBUG:teuthology.task.internal.check_lock:machine status is {'is_vm': False, 'locked': True, 'locked_since': '2021-05-07 18:55:31.202084', 'description': '/home/teuthworker/archive/sage-2021-05-07_18:25:10-upgrade:pacific-x:stress-split-wip-sage3-testing-2021-05-07-1141-distro-basic-smithi/6104808', 'locked_by': 'scheduled_sage@teuthology', 'up': True, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLzgC5uv0SZIuwXHdjf6TjnaY8bxwHShCQBGW5UhaCLpQFYGgvBZ85WVy6E1Q6YRN2WAU1BGocZLECU1b4GbRlI=', 'os_version': '8', 'machine_type': 'smithi', 'vm_host': None, 'os_type': 'centos', 'arch': 'x86_64', 'mac_address': None, 'name': 'smithi125.front.sepia.ceph.com'} 2021-05-07T19:00:34.869 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2021-05-07T19:00:34.880 INFO:teuthology.task.internal:roles: ubuntu@smithi058.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2021-05-07T19:00:34.880 INFO:teuthology.task.internal:roles: ubuntu@smithi125.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2021-05-07T19:00:34.880 INFO:teuthology.run_tasks:Running task console_log... 2021-05-07T19:00:34.941 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7ff2d2d52e18>, signals=[15]) 2021-05-07T19:00:34.943 INFO:teuthology.run_tasks:Running task internal.connect... 2021-05-07T19:00:34.953 INFO:teuthology.task.internal:Opening connections... 2021-05-07T19:00:34.953 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi058.front.sepia.ceph.com 2021-05-07T19:00:34.955 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi058.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2021-05-07T19:00:35.028 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi125.front.sepia.ceph.com 2021-05-07T19:00:35.029 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi125.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2021-05-07T19:00:35.097 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2021-05-07T19:00:35.107 DEBUG:teuthology.orchestra.run.smithi058:> uname -m 2021-05-07T19:00:35.142 INFO:teuthology.orchestra.run.smithi058.stdout:x86_64 2021-05-07T19:00:35.142 DEBUG:teuthology.orchestra.run.smithi058:> cat /etc/os-release 2021-05-07T19:00:35.196 INFO:teuthology.orchestra.run.smithi058.stdout:NAME="CentOS Linux" 2021-05-07T19:00:35.196 INFO:teuthology.orchestra.run.smithi058.stdout:VERSION="8 (Core)" 2021-05-07T19:00:35.197 INFO:teuthology.orchestra.run.smithi058.stdout:ID="centos" 2021-05-07T19:00:35.197 INFO:teuthology.orchestra.run.smithi058.stdout:ID_LIKE="rhel fedora" 2021-05-07T19:00:35.197 INFO:teuthology.orchestra.run.smithi058.stdout:VERSION_ID="8" 2021-05-07T19:00:35.197 INFO:teuthology.orchestra.run.smithi058.stdout:PLATFORM_ID="platform:el8" 2021-05-07T19:00:35.197 INFO:teuthology.orchestra.run.smithi058.stdout:PRETTY_NAME="CentOS Linux 8 (Core)" 2021-05-07T19:00:35.198 INFO:teuthology.orchestra.run.smithi058.stdout:ANSI_COLOR="0;31" 2021-05-07T19:00:35.198 INFO:teuthology.orchestra.run.smithi058.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2021-05-07T19:00:35.198 INFO:teuthology.orchestra.run.smithi058.stdout:HOME_URL="https://www.centos.org/" 2021-05-07T19:00:35.198 INFO:teuthology.orchestra.run.smithi058.stdout:BUG_REPORT_URL="https://bugs.centos.org/" 2021-05-07T19:00:35.198 INFO:teuthology.orchestra.run.smithi058.stdout: 2021-05-07T19:00:35.199 INFO:teuthology.orchestra.run.smithi058.stdout:CENTOS_MANTISBT_PROJECT="CentOS-8" 2021-05-07T19:00:35.199 INFO:teuthology.orchestra.run.smithi058.stdout:CENTOS_MANTISBT_PROJECT_VERSION="8" 2021-05-07T19:00:35.199 INFO:teuthology.orchestra.run.smithi058.stdout:REDHAT_SUPPORT_PRODUCT="centos" 2021-05-07T19:00:35.199 INFO:teuthology.orchestra.run.smithi058.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8" 2021-05-07T19:00:35.199 INFO:teuthology.orchestra.run.smithi058.stdout: 2021-05-07T19:00:35.200 INFO:teuthology.lock.ops:Updating smithi058.front.sepia.ceph.com on lock server 2021-05-07T19:00:35.216 DEBUG:teuthology.orchestra.run.smithi125:> uname -m 2021-05-07T19:00:35.232 INFO:teuthology.orchestra.run.smithi125.stdout:x86_64 2021-05-07T19:00:35.232 DEBUG:teuthology.orchestra.run.smithi125:> cat /etc/os-release 2021-05-07T19:00:35.286 INFO:teuthology.orchestra.run.smithi125.stdout:NAME="CentOS Linux" 2021-05-07T19:00:35.286 INFO:teuthology.orchestra.run.smithi125.stdout:VERSION="8 (Core)" 2021-05-07T19:00:35.286 INFO:teuthology.orchestra.run.smithi125.stdout:ID="centos" 2021-05-07T19:00:35.287 INFO:teuthology.orchestra.run.smithi125.stdout:ID_LIKE="rhel fedora" 2021-05-07T19:00:35.287 INFO:teuthology.orchestra.run.smithi125.stdout:VERSION_ID="8" 2021-05-07T19:00:35.287 INFO:teuthology.orchestra.run.smithi125.stdout:PLATFORM_ID="platform:el8" 2021-05-07T19:00:35.287 INFO:teuthology.orchestra.run.smithi125.stdout:PRETTY_NAME="CentOS Linux 8 (Core)" 2021-05-07T19:00:35.288 INFO:teuthology.orchestra.run.smithi125.stdout:ANSI_COLOR="0;31" 2021-05-07T19:00:35.288 INFO:teuthology.orchestra.run.smithi125.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2021-05-07T19:00:35.288 INFO:teuthology.orchestra.run.smithi125.stdout:HOME_URL="https://www.centos.org/" 2021-05-07T19:00:35.288 INFO:teuthology.orchestra.run.smithi125.stdout:BUG_REPORT_URL="https://bugs.centos.org/" 2021-05-07T19:00:35.288 INFO:teuthology.orchestra.run.smithi125.stdout: 2021-05-07T19:00:35.289 INFO:teuthology.orchestra.run.smithi125.stdout:CENTOS_MANTISBT_PROJECT="CentOS-8" 2021-05-07T19:00:35.289 INFO:teuthology.orchestra.run.smithi125.stdout:CENTOS_MANTISBT_PROJECT_VERSION="8" 2021-05-07T19:00:35.289 INFO:teuthology.orchestra.run.smithi125.stdout:REDHAT_SUPPORT_PRODUCT="centos" 2021-05-07T19:00:35.289 INFO:teuthology.orchestra.run.smithi125.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8" 2021-05-07T19:00:35.289 INFO:teuthology.orchestra.run.smithi125.stdout: 2021-05-07T19:00:35.290 INFO:teuthology.lock.ops:Updating smithi125.front.sepia.ceph.com on lock server 2021-05-07T19:00:35.305 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2021-05-07T19:00:35.318 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2021-05-07T19:00:35.330 INFO:teuthology.task.internal:Checking for old test directory... 2021-05-07T19:00:35.330 DEBUG:teuthology.orchestra.run.smithi058:> test '!' -e /home/ubuntu/cephtest 2021-05-07T19:00:35.333 DEBUG:teuthology.orchestra.run.smithi125:> test '!' -e /home/ubuntu/cephtest 2021-05-07T19:00:35.346 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2021-05-07T19:00:35.355 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2021-05-07T19:00:35.355 DEBUG:teuthology.orchestra.run.smithi058:> test -z $(ls -A /var/lib/ceph) 2021-05-07T19:00:35.387 DEBUG:teuthology.orchestra.run.smithi125:> test -z $(ls -A /var/lib/ceph) 2021-05-07T19:00:35.416 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2021-05-07T19:00:35.458 INFO:teuthology.run_tasks:Running task kernel... 2021-05-07T19:00:35.474 DEBUG:teuthology.task.kernel:normalized config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.c': {'kdb': True, 'sha1': 'distro'}, 'mgr.y': {'kdb': True, 'sha1': 'distro'}, 'osd.0': {'kdb': True, 'sha1': 'distro'}, 'osd.1': {'kdb': True, 'sha1': 'distro'}, 'osd.2': {'kdb': True, 'sha1': 'distro'}, 'osd.3': {'kdb': True, 'sha1': 'distro'}, 'client.0': {'kdb': True, 'sha1': 'distro'}, 'node-exporter.a': {'kdb': True, 'sha1': 'distro'}, 'alertmanager.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}, 'mgr.x': {'kdb': True, 'sha1': 'distro'}, 'osd.4': {'kdb': True, 'sha1': 'distro'}, 'osd.5': {'kdb': True, 'sha1': 'distro'}, 'osd.6': {'kdb': True, 'sha1': 'distro'}, 'osd.7': {'kdb': True, 'sha1': 'distro'}, 'client.1': {'kdb': True, 'sha1': 'distro'}, 'prometheus.a': {'kdb': True, 'sha1': 'distro'}, 'grafana.a': {'kdb': True, 'sha1': 'distro'}, 'node-exporter.b': {'kdb': True, 'sha1': 'distro'}} 2021-05-07T19:00:35.474 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2021-05-07T19:00:35.474 DEBUG:teuthology.orchestra.run.smithi058:> uname -r 2021-05-07T19:00:35.489 INFO:teuthology.orchestra.run.smithi058.stdout:4.18.0-193.28.1.el8_2.x86_64 2021-05-07T19:00:35.490 INFO:teuthology.task.kernel:Running kernel on smithi058: 4.18.0-193.28.1.el8_2.x86_64 2021-05-07T19:00:35.490 DEBUG:teuthology.orchestra.run.smithi058:> sudo yum install -y kernel 2021-05-07T19:00:38.162 INFO:teuthology.orchestra.run.smithi058.stdout:CentOS-8 - AppStream 28 MB/s | 6.3 MB 00:00 2021-05-07T19:00:39.350 INFO:teuthology.orchestra.run.smithi058.stdout:CentOS-8 - Base 24 MB/s | 2.3 MB 00:00 2021-05-07T19:00:39.828 INFO:teuthology.orchestra.run.smithi058.stdout:CentOS-8 - Extras 146 kB/s | 9.6 kB 00:00 2021-05-07T19:00:40.492 INFO:teuthology.orchestra.run.smithi058.stdout:Extra Packages for Enterprise Linux 15 MB/s | 9.3 MB 00:00 2021-05-07T19:00:42.660 INFO:teuthology.orchestra.run.smithi058.stdout:lab-extras 4.6 MB/s | 24 kB 00:00 2021-05-07T19:00:43.794 INFO:teuthology.orchestra.run.smithi058.stdout:Package kernel-4.18.0-193.el8.x86_64 is already installed. 2021-05-07T19:00:43.795 INFO:teuthology.orchestra.run.smithi058.stdout:Package kernel-4.18.0-193.28.1.el8_2.x86_64 is already installed. 2021-05-07T19:00:43.993 INFO:teuthology.orchestra.run.smithi058.stdout:Dependencies resolved. 2021-05-07T19:00:44.003 INFO:teuthology.orchestra.run.smithi058.stdout:================================================================================ 2021-05-07T19:00:44.003 INFO:teuthology.orchestra.run.smithi058.stdout: Package Arch Version Repository Size 2021-05-07T19:00:44.004 INFO:teuthology.orchestra.run.smithi058.stdout:================================================================================ 2021-05-07T19:00:44.004 INFO:teuthology.orchestra.run.smithi058.stdout:Installing: 2021-05-07T19:00:44.004 INFO:teuthology.orchestra.run.smithi058.stdout: kernel x86_64 4.18.0-240.22.1.el8_3 CentOS-Base 4.4 M 2021-05-07T19:00:44.004 INFO:teuthology.orchestra.run.smithi058.stdout:Upgrading: 2021-05-07T19:00:44.004 INFO:teuthology.orchestra.run.smithi058.stdout: linux-firmware noarch 20200619-101.git3890db36.el8_3 CentOS-Base 101 M 2021-05-07T19:00:44.005 INFO:teuthology.orchestra.run.smithi058.stdout:Installing dependencies: 2021-05-07T19:00:44.005 INFO:teuthology.orchestra.run.smithi058.stdout: kernel-core x86_64 4.18.0-240.22.1.el8_3 CentOS-Base 30 M 2021-05-07T19:00:44.005 INFO:teuthology.orchestra.run.smithi058.stdout: kernel-modules x86_64 4.18.0-240.22.1.el8_3 CentOS-Base 26 M 2021-05-07T19:00:44.005 INFO:teuthology.orchestra.run.smithi058.stdout: 2021-05-07T19:00:44.005 INFO:teuthology.orchestra.run.smithi058.stdout:Transaction Summary 2021-05-07T19:00:44.006 INFO:teuthology.orchestra.run.smithi058.stdout:================================================================================ 2021-05-07T19:00:44.006 INFO:teuthology.orchestra.run.smithi058.stdout:Install 3 Packages 2021-05-07T19:00:44.006 INFO:teuthology.orchestra.run.smithi058.stdout:Upgrade 1 Package 2021-05-07T19:00:44.006 INFO:teuthology.orchestra.run.smithi058.stdout: 2021-05-07T19:00:44.007 INFO:teuthology.orchestra.run.smithi058.stdout:Total download size: 162 M 2021-05-07T19:00:44.007 INFO:teuthology.orchestra.run.smithi058.stdout:Downloading Packages: 2021-05-07T19:00:44.279 INFO:teuthology.orchestra.run.smithi058.stdout:(1/4): kernel-4.18.0-240.22.1.el8_3.x86_64.rpm 16 MB/s | 4.4 MB 00:00 2021-05-07T19:00:44.770 INFO:teuthology.orchestra.run.smithi058.stdout:(2/4): kernel-modules-4.18.0-240.22.1.el8_3.x86 33 MB/s | 26 MB 00:00 2021-05-07T19:00:45.033 INFO:teuthology.orchestra.run.smithi058.stdout:(3/4): kernel-core-4.18.0-240.22.1.el8_3.x86_64 30 MB/s | 30 MB 00:01 2021-05-07T19:00:46.659 INFO:teuthology.orchestra.run.smithi058.stdout:(4/4): linux-firmware-20200619-101.git3890db36. 43 MB/s | 101 MB 00:02 2021-05-07T19:00:46.661 INFO:teuthology.orchestra.run.smithi058.stdout:-------------------------------------------------------------------------------- 2021-05-07T19:00:46.662 INFO:teuthology.orchestra.run.smithi058.stdout:Total 61 MB/s | 162 MB 00:02 2021-05-07T19:00:46.758 INFO:teuthology.orchestra.run.smithi058.stdout:Running transaction check 2021-05-07T19:00:47.074 INFO:teuthology.orchestra.run.smithi058.stdout:Transaction check succeeded. 2021-05-07T19:00:47.075 INFO:teuthology.orchestra.run.smithi058.stdout:Running transaction test 2021-05-07T19:00:48.787 INFO:teuthology.orchestra.run.smithi058.stdout:Transaction test succeeded. 2021-05-07T19:00:48.788 INFO:teuthology.orchestra.run.smithi058.stdout:Running transaction 2021-05-07T19:00:59.377 INFO:teuthology.orchestra.run.smithi058.stdout: Preparing : 1/1 2021-05-07T19:01:01.235 INFO:teuthology.orchestra.run.smithi058.stdout: Upgrading : linux-firmware-20200619-101.git3890db36.el8_3.noarch 1/5 2021-05-07T19:01:01.783 INFO:teuthology.orchestra.run.smithi058.stdout: Installing : kernel-core-4.18.0-240.22.1.el8_3.x86_64 2/5 2021-05-07T19:01:04.660 INFO:teuthology.orchestra.run.smithi058.stdout: Running scriptlet: kernel-core-4.18.0-240.22.1.el8_3.x86_64 2/5 2021-05-07T19:01:04.991 INFO:teuthology.orchestra.run.smithi058.stdout: Installing : kernel-modules-4.18.0-240.22.1.el8_3.x86_64 3/5 2021-05-07T19:01:08.089 INFO:teuthology.orchestra.run.smithi058.stdout: Running scriptlet: kernel-modules-4.18.0-240.22.1.el8_3.x86_64 3/5 2021-05-07T19:01:08.505 INFO:teuthology.orchestra.run.smithi058.stdout: Installing : kernel-4.18.0-240.22.1.el8_3.x86_64 4/5 2021-05-07T19:01:08.720 INFO:teuthology.orchestra.run.smithi058.stdout: Cleanup : linux-firmware-20191202-97.gite8a0f4c9.el8.noarch 5/5 2021-05-07T19:01:29.822 INFO:teuthology.orchestra.run.smithi058.stdout: Running scriptlet: kernel-core-4.18.0-240.22.1.el8_3.x86_64 5/5 2021-05-07T19:01:30.197 INFO:teuthology.orchestra.run.smithi058.stdout: Running scriptlet: linux-firmware-20191202-97.gite8a0f4c9.el8.noarch 5/5 2021-05-07T19:01:30.197 INFO:teuthology.orchestra.run.smithi058.stdout: Verifying : kernel-4.18.0-240.22.1.el8_3.x86_64 1/5 2021-05-07T19:01:30.198 INFO:teuthology.orchestra.run.smithi058.stdout: Verifying : kernel-core-4.18.0-240.22.1.el8_3.x86_64 2/5 2021-05-07T19:01:30.198 INFO:teuthology.orchestra.run.smithi058.stdout: Verifying : kernel-modules-4.18.0-240.22.1.el8_3.x86_64 3/5 2021-05-07T19:01:30.198 INFO:teuthology.orchestra.run.smithi058.stdout: Verifying : linux-firmware-20200619-101.git3890db36.el8_3.noarch 4/5 2021-05-07T19:01:30.626 INFO:teuthology.orchestra.run.smithi058.stdout: Verifying : linux-firmware-20191202-97.gite8a0f4c9.el8.noarch 5/5 2021-05-07T19:01:30.626 INFO:teuthology.orchestra.run.smithi058.stdout: 2021-05-07T19:01:30.626 INFO:teuthology.orchestra.run.smithi058.stdout:Upgraded: 2021-05-07T19:01:30.627 INFO:teuthology.orchestra.run.smithi058.stdout: linux-firmware-20200619-101.git3890db36.el8_3.noarch 2021-05-07T19:01:30.627 INFO:teuthology.orchestra.run.smithi058.stdout: 2021-05-07T19:01:30.627 INFO:teuthology.orchestra.run.smithi058.stdout:Installed: 2021-05-07T19:01:30.627 INFO:teuthology.orchestra.run.smithi058.stdout: kernel-4.18.0-240.22.1.el8_3.x86_64 2021-05-07T19:01:30.628 INFO:teuthology.orchestra.run.smithi058.stdout: kernel-core-4.18.0-240.22.1.el8_3.x86_64 2021-05-07T19:01:30.628 INFO:teuthology.orchestra.run.smithi058.stdout: kernel-modules-4.18.0-240.22.1.el8_3.x86_64 2021-05-07T19:01:30.628 INFO:teuthology.orchestra.run.smithi058.stdout: 2021-05-07T19:01:30.628 INFO:teuthology.orchestra.run.smithi058.stdout:Complete! 2021-05-07T19:01:30.787 DEBUG:teuthology.orchestra.run.smithi058:> rpm -q kernel --last | head -n 1 2021-05-07T19:01:30.903 INFO:teuthology.orchestra.run.smithi058.stdout:kernel-4.18.0-240.22.1.el8_3.x86_64 Fri 07 May 2021 07:01:08 PM UTC 2021-05-07T19:01:30.905 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-240.22.1.el8_3.x86_64 2021-05-07T19:01:30.905 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-193.28.1.el8_2.x86_64 Expected: 4.18.0-240.22.1.el8_3.x86_64 2021-05-07T19:01:30.905 DEBUG:teuthology.orchestra.run.smithi125:> uname -r 2021-05-07T19:01:30.923 INFO:teuthology.orchestra.run.smithi125.stdout:4.18.0-193.28.1.el8_2.x86_64 2021-05-07T19:01:30.923 INFO:teuthology.task.kernel:Running kernel on smithi125: 4.18.0-193.28.1.el8_2.x86_64 2021-05-07T19:01:30.924 DEBUG:teuthology.orchestra.run.smithi125:> sudo yum install -y kernel 2021-05-07T19:01:33.576 INFO:teuthology.orchestra.run.smithi125.stdout:CentOS-8 - AppStream 35 MB/s | 6.3 MB 00:00 2021-05-07T19:01:34.668 INFO:teuthology.orchestra.run.smithi125.stdout:CentOS-8 - Base 25 MB/s | 2.3 MB 00:00 2021-05-07T19:01:35.121 INFO:teuthology.orchestra.run.smithi125.stdout:CentOS-8 - Extras 131 kB/s | 9.6 kB 00:00 2021-05-07T19:01:35.858 INFO:teuthology.orchestra.run.smithi125.stdout:Extra Packages for Enterprise Linux 14 MB/s | 9.3 MB 00:00 2021-05-07T19:01:37.780 INFO:teuthology.orchestra.run.smithi125.stdout:lab-extras 4.1 MB/s | 24 kB 00:00 2021-05-07T19:01:38.837 INFO:teuthology.orchestra.run.smithi125.stdout:Package kernel-4.18.0-193.el8.x86_64 is already installed. 2021-05-07T19:01:38.837 INFO:teuthology.orchestra.run.smithi125.stdout:Package kernel-4.18.0-193.28.1.el8_2.x86_64 is already installed. 2021-05-07T19:01:39.022 INFO:teuthology.orchestra.run.smithi125.stdout:Dependencies resolved. 2021-05-07T19:01:39.031 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2021-05-07T19:01:39.031 INFO:teuthology.orchestra.run.smithi125.stdout: Package Arch Version Repository Size 2021-05-07T19:01:39.031 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2021-05-07T19:01:39.032 INFO:teuthology.orchestra.run.smithi125.stdout:Installing: 2021-05-07T19:01:39.032 INFO:teuthology.orchestra.run.smithi125.stdout: kernel x86_64 4.18.0-240.22.1.el8_3 CentOS-Base 4.4 M 2021-05-07T19:01:39.032 INFO:teuthology.orchestra.run.smithi125.stdout:Upgrading: 2021-05-07T19:01:39.032 INFO:teuthology.orchestra.run.smithi125.stdout: linux-firmware noarch 20200619-101.git3890db36.el8_3 CentOS-Base 101 M 2021-05-07T19:01:39.032 INFO:teuthology.orchestra.run.smithi125.stdout:Installing dependencies: 2021-05-07T19:01:39.033 INFO:teuthology.orchestra.run.smithi125.stdout: kernel-core x86_64 4.18.0-240.22.1.el8_3 CentOS-Base 30 M 2021-05-07T19:01:39.033 INFO:teuthology.orchestra.run.smithi125.stdout: kernel-modules x86_64 4.18.0-240.22.1.el8_3 CentOS-Base 26 M 2021-05-07T19:01:39.033 INFO:teuthology.orchestra.run.smithi125.stdout: 2021-05-07T19:01:39.033 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction Summary 2021-05-07T19:01:39.033 INFO:teuthology.orchestra.run.smithi125.stdout:================================================================================ 2021-05-07T19:01:39.034 INFO:teuthology.orchestra.run.smithi125.stdout:Install 3 Packages 2021-05-07T19:01:39.034 INFO:teuthology.orchestra.run.smithi125.stdout:Upgrade 1 Package 2021-05-07T19:01:39.034 INFO:teuthology.orchestra.run.smithi125.stdout: 2021-05-07T19:01:39.034 INFO:teuthology.orchestra.run.smithi125.stdout:Total download size: 162 M 2021-05-07T19:01:39.034 INFO:teuthology.orchestra.run.smithi125.stdout:Downloading Packages: 2021-05-07T19:01:39.167 INFO:teuthology.orchestra.run.smithi125.stdout:(1/4): kernel-4.18.0-240.22.1.el8_3.x86_64.rpm 32 MB/s | 4.4 MB 00:00 2021-05-07T19:01:40.807 INFO:teuthology.orchestra.run.smithi125.stdout:(2/4): kernel-core-4.18.0-240.22.1.el8_3.x86_64 17 MB/s | 30 MB 00:01 2021-05-07T19:01:40.997 INFO:teuthology.orchestra.run.smithi125.stdout:(3/4): kernel-modules-4.18.0-240.22.1.el8_3.x86 13 MB/s | 26 MB 00:01 2021-05-07T19:01:41.988 INFO:teuthology.orchestra.run.smithi125.stdout:(4/4): linux-firmware-20200619-101.git3890db36. 36 MB/s | 101 MB 00:02 2021-05-07T19:01:41.992 INFO:teuthology.orchestra.run.smithi125.stdout:-------------------------------------------------------------------------------- 2021-05-07T19:01:41.992 INFO:teuthology.orchestra.run.smithi125.stdout:Total 55 MB/s | 162 MB 00:02 2021-05-07T19:01:42.083 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction check 2021-05-07T19:01:42.357 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction check succeeded. 2021-05-07T19:01:42.357 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction test 2021-05-07T19:01:44.011 INFO:teuthology.orchestra.run.smithi125.stdout:Transaction test succeeded. 2021-05-07T19:01:44.013 INFO:teuthology.orchestra.run.smithi125.stdout:Running transaction 2021-05-07T19:01:53.957 INFO:teuthology.orchestra.run.smithi125.stdout: Preparing : 1/1 2021-05-07T19:01:55.735 INFO:teuthology.orchestra.run.smithi125.stdout: Upgrading : linux-firmware-20200619-101.git3890db36.el8_3.noarch 1/5 2021-05-07T19:01:56.331 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : kernel-core-4.18.0-240.22.1.el8_3.x86_64 2/5 2021-05-07T19:01:58.972 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: kernel-core-4.18.0-240.22.1.el8_3.x86_64 2/5 2021-05-07T19:01:59.305 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : kernel-modules-4.18.0-240.22.1.el8_3.x86_64 3/5 2021-05-07T19:02:02.183 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: kernel-modules-4.18.0-240.22.1.el8_3.x86_64 3/5 2021-05-07T19:02:02.362 INFO:teuthology.orchestra.run.smithi125.stdout: Installing : kernel-4.18.0-240.22.1.el8_3.x86_64 4/5 2021-05-07T19:02:02.590 INFO:teuthology.orchestra.run.smithi125.stdout: Cleanup : linux-firmware-20191202-97.gite8a0f4c9.el8.noarch 5/5 2021-05-07T19:02:22.216 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: kernel-core-4.18.0-240.22.1.el8_3.x86_64 5/5 2021-05-07T19:02:22.890 INFO:teuthology.orchestra.run.smithi125.stdout: Running scriptlet: linux-firmware-20191202-97.gite8a0f4c9.el8.noarch 5/5 2021-05-07T19:02:22.890 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : kernel-4.18.0-240.22.1.el8_3.x86_64 1/5 2021-05-07T19:02:22.890 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : kernel-core-4.18.0-240.22.1.el8_3.x86_64 2/5 2021-05-07T19:02:22.891 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : kernel-modules-4.18.0-240.22.1.el8_3.x86_64 3/5 2021-05-07T19:02:22.891 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : linux-firmware-20200619-101.git3890db36.el8_3.noarch 4/5 2021-05-07T19:02:23.347 INFO:teuthology.orchestra.run.smithi125.stdout: Verifying : linux-firmware-20191202-97.gite8a0f4c9.el8.noarch 5/5 2021-05-07T19:02:23.348 INFO:teuthology.orchestra.run.smithi125.stdout: 2021-05-07T19:02:23.348 INFO:teuthology.orchestra.run.smithi125.stdout:Upgraded: 2021-05-07T19:02:23.348 INFO:teuthology.orchestra.run.smithi125.stdout: linux-firmware-20200619-101.git3890db36.el8_3.noarch 2021-05-07T19:02:23.348 INFO:teuthology.orchestra.run.smithi125.stdout: 2021-05-07T19:02:23.349 INFO:teuthology.orchestra.run.smithi125.stdout:Installed: 2021-05-07T19:02:23.349 INFO:teuthology.orchestra.run.smithi125.stdout: kernel-4.18.0-240.22.1.el8_3.x86_64 2021-05-07T19:02:23.349 INFO:teuthology.orchestra.run.smithi125.stdout: kernel-core-4.18.0-240.22.1.el8_3.x86_64 2021-05-07T19:02:23.349 INFO:teuthology.orchestra.run.smithi125.stdout: kernel-modules-4.18.0-240.22.1.el8_3.x86_64 2021-05-07T19:02:23.349 INFO:teuthology.orchestra.run.smithi125.stdout: 2021-05-07T19:02:23.350 INFO:teuthology.orchestra.run.smithi125.stdout:Complete! 2021-05-07T19:02:23.483 DEBUG:teuthology.orchestra.run.smithi125:> rpm -q kernel --last | head -n 1 2021-05-07T19:02:23.599 INFO:teuthology.orchestra.run.smithi125.stdout:kernel-4.18.0-240.22.1.el8_3.x86_64 Fri 07 May 2021 07:02:02 PM UTC 2021-05-07T19:02:23.601 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-240.22.1.el8_3.x86_64 2021-05-07T19:02:23.601 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-193.28.1.el8_2.x86_64 Expected: 4.18.0-240.22.1.el8_3.x86_64 2021-05-07T19:02:23.601 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2021-05-07T19:02:23.601 DEBUG:teuthology.task.kernel:src is distro, skipping download 2021-05-07T19:02:23.602 DEBUG:teuthology.task.kernel:src is distro, skipping download 2021-05-07T19:02:23.602 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2021-05-07T19:02:23.602 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi058.front.sepia.ceph.com, path=None, version=distro) 2021-05-07T19:02:23.603 DEBUG:teuthology.orchestra.run.smithi058:> rpm -q kernel --last | head -n 1 2021-05-07T19:02:23.704 INFO:teuthology.orchestra.run.smithi058.stdout:kernel-4.18.0-240.22.1.el8_3.x86_64 Fri 07 May 2021 07:01:08 PM UTC 2021-05-07T19:02:23.705 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-240.22.1.el8_3.x86_64 2021-05-07T19:02:23.706 DEBUG:teuthology.orchestra.run.smithi058:> sudo rpm -qi grub2-tools 2021-05-07T19:02:23.738 INFO:teuthology.orchestra.run.smithi058.stdout:Name : grub2-tools 2021-05-07T19:02:23.739 INFO:teuthology.orchestra.run.smithi058.stdout:Epoch : 1 2021-05-07T19:02:23.739 INFO:teuthology.orchestra.run.smithi058.stdout:Version : 2.02 2021-05-07T19:02:23.739 INFO:teuthology.orchestra.run.smithi058.stdout:Release : 81.el8 2021-05-07T19:02:23.740 INFO:teuthology.orchestra.run.smithi058.stdout:Architecture: x86_64 2021-05-07T19:02:23.740 INFO:teuthology.orchestra.run.smithi058.stdout:Install Date: Fri 13 Nov 2020 03:02:55 PM UTC 2021-05-07T19:02:23.740 INFO:teuthology.orchestra.run.smithi058.stdout:Group : System Environment/Base 2021-05-07T19:02:23.740 INFO:teuthology.orchestra.run.smithi058.stdout:Size : 9192705 2021-05-07T19:02:23.741 INFO:teuthology.orchestra.run.smithi058.stdout:License : GPLv3+ 2021-05-07T19:02:23.741 INFO:teuthology.orchestra.run.smithi058.stdout:Signature : RSA/SHA256, Tue 14 Apr 2020 03:18:37 PM UTC, Key ID 05b555b38483c65d 2021-05-07T19:02:23.741 INFO:teuthology.orchestra.run.smithi058.stdout:Source RPM : grub2-2.02-81.el8.src.rpm 2021-05-07T19:02:23.741 INFO:teuthology.orchestra.run.smithi058.stdout:Build Date : Tue 14 Apr 2020 02:54:04 PM UTC 2021-05-07T19:02:23.741 INFO:teuthology.orchestra.run.smithi058.stdout:Build Host : kbuilder.bsys.centos.org 2021-05-07T19:02:23.742 INFO:teuthology.orchestra.run.smithi058.stdout:Relocations : (not relocatable) 2021-05-07T19:02:23.742 INFO:teuthology.orchestra.run.smithi058.stdout:Packager : CentOS BuildSystem 2021-05-07T19:02:23.742 INFO:teuthology.orchestra.run.smithi058.stdout:Vendor : CentOS 2021-05-07T19:02:23.742 INFO:teuthology.orchestra.run.smithi058.stdout:URL : http://www.gnu.org/software/grub/ 2021-05-07T19:02:23.742 INFO:teuthology.orchestra.run.smithi058.stdout:Summary : Support tools for GRUB. 2021-05-07T19:02:23.743 INFO:teuthology.orchestra.run.smithi058.stdout:Description : 2021-05-07T19:02:23.743 INFO:teuthology.orchestra.run.smithi058.stdout: 2021-05-07T19:02:23.743 INFO:teuthology.orchestra.run.smithi058.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2021-05-07T19:02:23.743 INFO:teuthology.orchestra.run.smithi058.stdout:customizable bootloader with modular architecture. It supports a rich 2021-05-07T19:02:23.744 INFO:teuthology.orchestra.run.smithi058.stdout:variety of kernel formats, file systems, computer architectures and 2021-05-07T19:02:23.744 INFO:teuthology.orchestra.run.smithi058.stdout:hardware devices. 2021-05-07T19:02:23.744 INFO:teuthology.orchestra.run.smithi058.stdout: 2021-05-07T19:02:23.744 INFO:teuthology.orchestra.run.smithi058.stdout:This subpackage provides tools for support of all platforms. 2021-05-07T19:02:23.745 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2021-05-07T19:02:23.746 INFO:teuthology.task.kernel:Updating grub on smithi058 to boot 4.18.0-240.22.1.el8_3.x86_64 2021-05-07T19:02:23.746 DEBUG:teuthology.orchestra.run.smithi058:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2021-05-07T19:02:24.417 INFO:teuthology.orchestra.run.smithi058.stderr:Generating grub configuration file ... 2021-05-07T19:02:26.042 INFO:teuthology.orchestra.run.smithi058.stderr:done 2021-05-07T19:02:26.045 DEBUG:teuthology.orchestra.run.smithi058:> mktemp 2021-05-07T19:02:26.060 INFO:teuthology.orchestra.run.smithi058.stdout:/tmp/tmp.MUn6N2GfG2 2021-05-07T19:02:26.060 DEBUG:teuthology.orchestra.run.smithi058:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.MUn6N2GfG2 2021-05-07T19:02:26.130 DEBUG:teuthology.orchestra.run.smithi058:> sudo chmod 0666 /tmp/tmp.MUn6N2GfG2 2021-05-07T19:02:26.290 DEBUG:teuthology.orchestra.remote:smithi058:/tmp/tmp.MUn6N2GfG2 is 4KB 2021-05-07T19:02:26.340 DEBUG:teuthology.orchestra.run.smithi058:> rm -fr /tmp/tmp.MUn6N2GfG2 2021-05-07T19:02:26.354 DEBUG:teuthology.orchestra.run.smithi058:> sudo /bin/ls /boot/loader/entries || true 2021-05-07T19:02:26.417 INFO:teuthology.orchestra.run.smithi058.stdout:2951a06464bf41f6a3bf91594147ecff-0-rescue.conf 2021-05-07T19:02:26.418 INFO:teuthology.orchestra.run.smithi058.stdout:2951a06464bf41f6a3bf91594147ecff-4.18.0-193.28.1.el8_2.x86_64.conf 2021-05-07T19:02:26.418 INFO:teuthology.orchestra.run.smithi058.stdout:2951a06464bf41f6a3bf91594147ecff-4.18.0-193.el8.x86_64.conf 2021-05-07T19:02:26.418 INFO:teuthology.orchestra.run.smithi058.stdout:2951a06464bf41f6a3bf91594147ecff-4.18.0-240.22.1.el8_3.x86_64.conf 2021-05-07T19:02:26.419 DEBUG:teuthology.orchestra.run.smithi058:> sudo grub2-set-default 2951a06464bf41f6a3bf91594147ecff-4.18.0-240.22.1.el8_3.x86_64 2021-05-07T19:02:26.586 DEBUG:teuthology.orchestra.run.smithi058:> sudo shutdown -r now 2021-05-07T19:02:26.588 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2021-05-07T19:02:26.588 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi125.front.sepia.ceph.com, path=None, version=distro) 2021-05-07T19:02:26.589 DEBUG:teuthology.orchestra.run.smithi125:> rpm -q kernel --last | head -n 1 2021-05-07T19:02:26.684 INFO:teuthology.orchestra.run.smithi125.stdout:kernel-4.18.0-240.22.1.el8_3.x86_64 Fri 07 May 2021 07:02:02 PM UTC 2021-05-07T19:02:26.686 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-240.22.1.el8_3.x86_64 2021-05-07T19:02:26.687 DEBUG:teuthology.orchestra.run.smithi125:> sudo rpm -qi grub2-tools 2021-05-07T19:02:26.717 INFO:teuthology.orchestra.run.smithi125.stdout:Name : grub2-tools 2021-05-07T19:02:26.718 INFO:teuthology.orchestra.run.smithi125.stdout:Epoch : 1 2021-05-07T19:02:26.718 INFO:teuthology.orchestra.run.smithi125.stdout:Version : 2.02 2021-05-07T19:02:26.718 INFO:teuthology.orchestra.run.smithi125.stdout:Release : 81.el8 2021-05-07T19:02:26.719 INFO:teuthology.orchestra.run.smithi125.stdout:Architecture: x86_64 2021-05-07T19:02:26.719 INFO:teuthology.orchestra.run.smithi125.stdout:Install Date: Fri 13 Nov 2020 03:02:55 PM UTC 2021-05-07T19:02:26.719 INFO:teuthology.orchestra.run.smithi125.stdout:Group : System Environment/Base 2021-05-07T19:02:26.719 INFO:teuthology.orchestra.run.smithi125.stdout:Size : 9192705 2021-05-07T19:02:26.720 INFO:teuthology.orchestra.run.smithi125.stdout:License : GPLv3+ 2021-05-07T19:02:26.720 INFO:teuthology.orchestra.run.smithi125.stdout:Signature : RSA/SHA256, Tue 14 Apr 2020 03:18:37 PM UTC, Key ID 05b555b38483c65d 2021-05-07T19:02:26.720 INFO:teuthology.orchestra.run.smithi125.stdout:Source RPM : grub2-2.02-81.el8.src.rpm 2021-05-07T19:02:26.720 INFO:teuthology.orchestra.run.smithi125.stdout:Build Date : Tue 14 Apr 2020 02:54:04 PM UTC 2021-05-07T19:02:26.720 INFO:teuthology.orchestra.run.smithi125.stdout:Build Host : kbuilder.bsys.centos.org 2021-05-07T19:02:26.721 INFO:teuthology.orchestra.run.smithi125.stdout:Relocations : (not relocatable) 2021-05-07T19:02:26.721 INFO:teuthology.orchestra.run.smithi125.stdout:Packager : CentOS BuildSystem 2021-05-07T19:02:26.721 INFO:teuthology.orchestra.run.smithi125.stdout:Vendor : CentOS 2021-05-07T19:02:26.721 INFO:teuthology.orchestra.run.smithi125.stdout:URL : http://www.gnu.org/software/grub/ 2021-05-07T19:02:26.721 INFO:teuthology.orchestra.run.smithi125.stdout:Summary : Support tools for GRUB. 2021-05-07T19:02:26.722 INFO:teuthology.orchestra.run.smithi125.stdout:Description : 2021-05-07T19:02:26.722 INFO:teuthology.orchestra.run.smithi125.stdout: 2021-05-07T19:02:26.722 INFO:teuthology.orchestra.run.smithi125.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2021-05-07T19:02:26.722 INFO:teuthology.orchestra.run.smithi125.stdout:customizable bootloader with modular architecture. It supports a rich 2021-05-07T19:02:26.723 INFO:teuthology.orchestra.run.smithi125.stdout:variety of kernel formats, file systems, computer architectures and 2021-05-07T19:02:26.723 INFO:teuthology.orchestra.run.smithi125.stdout:hardware devices. 2021-05-07T19:02:26.723 INFO:teuthology.orchestra.run.smithi125.stdout: 2021-05-07T19:02:26.723 INFO:teuthology.orchestra.run.smithi125.stdout:This subpackage provides tools for support of all platforms. 2021-05-07T19:02:26.724 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2021-05-07T19:02:26.725 INFO:teuthology.task.kernel:Updating grub on smithi125 to boot 4.18.0-240.22.1.el8_3.x86_64 2021-05-07T19:02:26.725 DEBUG:teuthology.orchestra.run.smithi125:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2021-05-07T19:02:28.465 INFO:teuthology.orchestra.run.smithi125.stderr:Generating grub configuration file ... 2021-05-07T19:02:30.164 INFO:teuthology.orchestra.run.smithi125.stderr:done 2021-05-07T19:02:30.166 DEBUG:teuthology.orchestra.run.smithi125:> mktemp 2021-05-07T19:02:30.191 INFO:teuthology.orchestra.run.smithi125.stdout:/tmp/tmp.ChY5yj82s5 2021-05-07T19:02:30.191 DEBUG:teuthology.orchestra.run.smithi125:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.ChY5yj82s5 2021-05-07T19:02:30.261 DEBUG:teuthology.orchestra.run.smithi125:> sudo chmod 0666 /tmp/tmp.ChY5yj82s5 2021-05-07T19:02:30.403 DEBUG:teuthology.orchestra.remote:smithi125:/tmp/tmp.ChY5yj82s5 is 4KB 2021-05-07T19:02:30.450 DEBUG:teuthology.orchestra.run.smithi125:> rm -fr /tmp/tmp.ChY5yj82s5 2021-05-07T19:02:30.464 DEBUG:teuthology.orchestra.run.smithi125:> sudo /bin/ls /boot/loader/entries || true 2021-05-07T19:02:30.527 INFO:teuthology.orchestra.run.smithi125.stdout:2951a06464bf41f6a3bf91594147ecff-0-rescue.conf 2021-05-07T19:02:30.527 INFO:teuthology.orchestra.run.smithi125.stdout:2951a06464bf41f6a3bf91594147ecff-4.18.0-193.28.1.el8_2.x86_64.conf 2021-05-07T19:02:30.527 INFO:teuthology.orchestra.run.smithi125.stdout:2951a06464bf41f6a3bf91594147ecff-4.18.0-193.el8.x86_64.conf 2021-05-07T19:02:30.527 INFO:teuthology.orchestra.run.smithi125.stdout:2951a06464bf41f6a3bf91594147ecff-4.18.0-240.22.1.el8_3.x86_64.conf 2021-05-07T19:02:30.528 DEBUG:teuthology.orchestra.run.smithi125:> sudo grub2-set-default 2951a06464bf41f6a3bf91594147ecff-4.18.0-240.22.1.el8_3.x86_64 2021-05-07T19:02:30.679 DEBUG:teuthology.orchestra.run.smithi125:> sudo shutdown -r now 2021-05-07T19:03:00.713 INFO:teuthology.misc:Re-opening connections... 2021-05-07T19:03:00.713 INFO:teuthology.misc:trying to connect to ubuntu@smithi058.front.sepia.ceph.com 2021-05-07T19:03:00.714 INFO:teuthology.orchestra.remote:Trying to reconnect to host 2021-05-07T19:03:00.714 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi058.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2021-05-07T19:03:49.088 DEBUG:teuthology.exit:Got signal 15; running 1 handler... 2021-05-07T19:03:49.089 DEBUG:teuthology.task.console_log:Killing console logger for smithi058 2021-05-07T19:03:49.090 DEBUG:teuthology.task.console_log:Killing console logger for smithi125 2021-05-07T19:03:49.090 DEBUG:teuthology.exit:Finished running handlers