2024-04-10T22:37:07.158 INFO:root:teuthology version: 0.0.1.dev267+g6c63784 2024-04-10T22:37:07.158 DEBUG:teuthology.run:Teuthology command: teuthology --owner scheduled_teuthology@teuthology --name teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi --verbose --archive /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650194 --description upgrade:pacific-x/stress-split/{0-distro/centos_8.stream_container_tools_crun 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} -- /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650194/orig.config.yaml 2024-04-10T22:37:07.214 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-10T22:37:07.313 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650194 branch: reef description: upgrade:pacific-x/stress-split/{0-distro/centos_8.stream_container_tools_crun 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} email: ceph-qa@ceph.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' first_in_suite: false job_id: '7650194' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run basic import/export cli tests for rbd ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 8.stream overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: reef sha1: 4f70a9a90881107996c8b7a3e89ba8b32624713c owner: scheduled_teuthology@teuthology priority: 830 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 4604 sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: pacific clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 0/1 suite: upgrade:pacific-x suite_branch: reef suite_path: /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 4f70a9a90881107996c8b7a3e89ba8b32624713c targets: smithi102.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAYn7c2irT2HVTr9SoxIe4TY/qZoOOBNNp+2Avooi8GZGS39lhLelR8BlBGukMHHBw0wp/uClqWiy6fjqKVuyTY= smithi138.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNI9I97+Z3WbG7lfx7sIKyYunYVgYJqMabmJeNdERfodMr6K9VDGoAm7Usm3pZlIa/DCRqG5FIepX+SoaUNacwQ= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - sudo sed -i 's/runtime = "runc"/#runtime = "runc"/g' /usr/share/containers/containers.conf - sudo sed -i 's/#runtime = "crun"/runtime = "crun"/g' /usr/share/containers/containers.conf - install: branch: pacific exclude_packages: - ceph-volume - cephadm: cephadm_branch: pacific cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:pacific - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 6c637841c215537a4502385240412f1966e0faab timestamp: 2024-04-10_01:08:08 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-04-10T22:37:07.313 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa; will attempt to use it 2024-04-10T22:37:07.314 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa/tasks 2024-04-10T22:37:07.314 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-10T22:37:07.316 INFO:teuthology.task.internal:Checking packages... 2024-04-10T22:37:07.340 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'd540ebaca6b131a1dd560e7f69e024b133bbaa42' 2024-04-10T22:37:07.340 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-10T22:37:07.340 INFO:teuthology.packaging:ref: None 2024-04-10T22:37:07.340 INFO:teuthology.packaging:tag: None 2024-04-10T22:37:07.340 INFO:teuthology.packaging:branch: reef 2024-04-10T22:37:07.340 INFO:teuthology.packaging:sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 2024-04-10T22:37:07.340 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef 2024-04-10T22:37:07.668 INFO:teuthology.task.internal:Found packages for ceph version 18.2.2-1189.gc96c8d47 2024-04-10T22:37:07.670 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-10T22:37:07.677 INFO:teuthology.task.internal:no buildpackages task found 2024-04-10T22:37:07.677 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-10T22:37:07.684 INFO:teuthology.task.internal:Saving configuration 2024-04-10T22:37:07.706 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-10T22:37:07.712 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T22:37:07.739 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi102.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650194', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:21.306042', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAYn7c2irT2HVTr9SoxIe4TY/qZoOOBNNp+2Avooi8GZGS39lhLelR8BlBGukMHHBw0wp/uClqWiy6fjqKVuyTY='} 2024-04-10T22:37:07.762 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi138.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650194', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:21.307848', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNI9I97+Z3WbG7lfx7sIKyYunYVgYJqMabmJeNdERfodMr6K9VDGoAm7Usm3pZlIa/DCRqG5FIepX+SoaUNacwQ='} 2024-04-10T22:37:07.762 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-10T22:37:07.768 INFO:teuthology.task.internal:roles: ubuntu@smithi102.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-04-10T22:37:07.768 INFO:teuthology.task.internal:roles: ubuntu@smithi138.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-04-10T22:37:07.769 INFO:teuthology.run_tasks:Running task console_log... 2024-04-10T22:37:07.841 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f5042f8a0d0>, signals=[15]) 2024-04-10T22:37:07.841 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-10T22:37:07.847 INFO:teuthology.task.internal:Opening connections... 2024-04-10T22:37:07.848 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi102.front.sepia.ceph.com 2024-04-10T22:37:07.850 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi102.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:37:07.923 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi138.front.sepia.ceph.com 2024-04-10T22:37:07.923 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi138.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:37:07.996 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-10T22:37:08.003 DEBUG:teuthology.orchestra.run.smithi102:> uname -m 2024-04-10T22:37:08.025 INFO:teuthology.orchestra.run.smithi102.stdout:x86_64 2024-04-10T22:37:08.025 DEBUG:teuthology.orchestra.run.smithi102:> cat /etc/os-release 2024-04-10T22:37:08.086 INFO:teuthology.orchestra.run.smithi102.stdout:NAME="CentOS Stream" 2024-04-10T22:37:08.086 INFO:teuthology.orchestra.run.smithi102.stdout:VERSION="8" 2024-04-10T22:37:08.087 INFO:teuthology.orchestra.run.smithi102.stdout:ID="centos" 2024-04-10T22:37:08.087 INFO:teuthology.orchestra.run.smithi102.stdout:ID_LIKE="rhel fedora" 2024-04-10T22:37:08.087 INFO:teuthology.orchestra.run.smithi102.stdout:VERSION_ID="8" 2024-04-10T22:37:08.087 INFO:teuthology.orchestra.run.smithi102.stdout:PLATFORM_ID="platform:el8" 2024-04-10T22:37:08.087 INFO:teuthology.orchestra.run.smithi102.stdout:PRETTY_NAME="CentOS Stream 8" 2024-04-10T22:37:08.087 INFO:teuthology.orchestra.run.smithi102.stdout:ANSI_COLOR="0;31" 2024-04-10T22:37:08.087 INFO:teuthology.orchestra.run.smithi102.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2024-04-10T22:37:08.087 INFO:teuthology.orchestra.run.smithi102.stdout:HOME_URL="https://centos.org/" 2024-04-10T22:37:08.087 INFO:teuthology.orchestra.run.smithi102.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-10T22:37:08.087 INFO:teuthology.orchestra.run.smithi102.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2024-04-10T22:37:08.087 INFO:teuthology.orchestra.run.smithi102.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-10T22:37:08.088 INFO:teuthology.lock.ops:Updating smithi102.front.sepia.ceph.com on lock server 2024-04-10T22:37:08.105 DEBUG:teuthology.orchestra.run.smithi138:> uname -m 2024-04-10T22:37:08.127 INFO:teuthology.orchestra.run.smithi138.stdout:x86_64 2024-04-10T22:37:08.127 DEBUG:teuthology.orchestra.run.smithi138:> cat /etc/os-release 2024-04-10T22:37:08.188 INFO:teuthology.orchestra.run.smithi138.stdout:NAME="CentOS Stream" 2024-04-10T22:37:08.188 INFO:teuthology.orchestra.run.smithi138.stdout:VERSION="8" 2024-04-10T22:37:08.188 INFO:teuthology.orchestra.run.smithi138.stdout:ID="centos" 2024-04-10T22:37:08.188 INFO:teuthology.orchestra.run.smithi138.stdout:ID_LIKE="rhel fedora" 2024-04-10T22:37:08.188 INFO:teuthology.orchestra.run.smithi138.stdout:VERSION_ID="8" 2024-04-10T22:37:08.188 INFO:teuthology.orchestra.run.smithi138.stdout:PLATFORM_ID="platform:el8" 2024-04-10T22:37:08.189 INFO:teuthology.orchestra.run.smithi138.stdout:PRETTY_NAME="CentOS Stream 8" 2024-04-10T22:37:08.189 INFO:teuthology.orchestra.run.smithi138.stdout:ANSI_COLOR="0;31" 2024-04-10T22:37:08.189 INFO:teuthology.orchestra.run.smithi138.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2024-04-10T22:37:08.189 INFO:teuthology.orchestra.run.smithi138.stdout:HOME_URL="https://centos.org/" 2024-04-10T22:37:08.189 INFO:teuthology.orchestra.run.smithi138.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-10T22:37:08.189 INFO:teuthology.orchestra.run.smithi138.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2024-04-10T22:37:08.189 INFO:teuthology.orchestra.run.smithi138.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-10T22:37:08.189 INFO:teuthology.lock.ops:Updating smithi138.front.sepia.ceph.com on lock server 2024-04-10T22:37:08.213 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-10T22:37:08.222 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-10T22:37:08.227 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-10T22:37:08.227 DEBUG:teuthology.orchestra.run.smithi102:> test '!' -e /home/ubuntu/cephtest 2024-04-10T22:37:08.229 DEBUG:teuthology.orchestra.run.smithi138:> test '!' -e /home/ubuntu/cephtest 2024-04-10T22:37:08.249 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-10T22:37:08.256 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-10T22:37:08.256 DEBUG:teuthology.orchestra.run.smithi102:> test -z $(ls -A /var/lib/ceph) 2024-04-10T22:37:08.291 DEBUG:teuthology.orchestra.run.smithi138:> test -z $(ls -A /var/lib/ceph) 2024-04-10T22:37:08.326 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-10T22:37:08.382 INFO:teuthology.run_tasks:Running task kernel... 2024-04-10T22:37:08.396 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-04-10T22:37:08.396 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-04-10T22:37:08.396 DEBUG:teuthology.orchestra.run.smithi102:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-10T22:37:08.397 DEBUG:teuthology.orchestra.run.smithi138:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-10T22:37:08.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:37:08.417 DEBUG:teuthology.orchestra.run.smithi102:> uname -r 2024-04-10T22:37:08.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:37:08.418 DEBUG:teuthology.orchestra.run.smithi138:> uname -r 2024-04-10T22:37:08.479 INFO:teuthology.orchestra.run.smithi102.stdout:4.18.0-489.el8.x86_64 2024-04-10T22:37:08.479 INFO:teuthology.task.kernel:Running kernel on smithi102: 4.18.0-489.el8.x86_64 2024-04-10T22:37:08.479 DEBUG:teuthology.orchestra.run.smithi102:> sudo yum install -y kernel 2024-04-10T22:37:08.480 INFO:teuthology.orchestra.run.smithi138.stdout:4.18.0-489.el8.x86_64 2024-04-10T22:37:08.480 INFO:teuthology.task.kernel:Running kernel on smithi138: 4.18.0-489.el8.x86_64 2024-04-10T22:37:08.480 DEBUG:teuthology.orchestra.run.smithi138:> sudo yum install -y kernel 2024-04-10T22:37:12.559 INFO:teuthology.orchestra.run.smithi102.stdout:CentOS Stream 8 - AppStream 20 MB/s | 28 MB 00:01 2024-04-10T22:37:18.909 INFO:teuthology.orchestra.run.smithi102.stdout:CentOS Stream 8 - BaseOS 8.2 MB/s | 10 MB 00:01 2024-04-10T22:39:24.349 INFO:teuthology.orchestra.run.smithi138.stdout:CentOS Stream 8 - AppStream 214 kB/s | 28 MB 02:13 2024-04-10T22:39:30.897 INFO:teuthology.orchestra.run.smithi102.stdout:CentOS Stream 8 - Extras 140 B/s | 18 kB 02:10 2024-04-10T22:39:31.380 INFO:teuthology.orchestra.run.smithi102.stdout:CentOS Stream 8 - Extras common packages 21 kB/s | 7.7 kB 00:00 2024-04-10T22:39:32.288 INFO:teuthology.orchestra.run.smithi138.stdout:CentOS Stream 8 - BaseOS 3.5 MB/s | 10 MB 00:02 2024-04-10T22:39:34.372 INFO:teuthology.orchestra.run.smithi138.stdout:CentOS Stream 8 - Extras 38 kB/s | 18 kB 00:00 2024-04-10T22:40:15.757 INFO:teuthology.orchestra.run.smithi102.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:44 2024-04-10T22:40:15.758 INFO:teuthology.orchestra.run.smithi102.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:40:15.758 INFO:teuthology.orchestra.run.smithi102.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:40:15.758 INFO:teuthology.orchestra.run.smithi102.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:40:23.019 INFO:teuthology.orchestra.run.smithi102.stdout:Extra Packages for Enterprise Linux 2.2 MB/s | 16 MB 00:07 2024-04-10T22:40:26.743 INFO:teuthology.orchestra.run.smithi102.stdout:lab-extras 577 kB/s | 24 kB 00:00 2024-04-10T22:40:26.886 INFO:teuthology.orchestra.run.smithi102.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:40:29.714 INFO:teuthology.orchestra.run.smithi102.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2024-04-10T22:40:29.714 INFO:teuthology.orchestra.run.smithi102.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2024-04-10T22:40:29.841 INFO:teuthology.orchestra.run.smithi102.stdout:Dependencies resolved. 2024-04-10T22:40:29.842 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T22:40:29.842 INFO:teuthology.orchestra.run.smithi102.stdout: Package Architecture Version Repository Size 2024-04-10T22:40:29.842 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T22:40:29.843 INFO:teuthology.orchestra.run.smithi102.stdout:Installing: 2024-04-10T22:40:29.843 INFO:teuthology.orchestra.run.smithi102.stdout: kernel x86_64 4.18.0-552.el8 baseos 10 M 2024-04-10T22:40:29.843 INFO:teuthology.orchestra.run.smithi102.stdout:Installing dependencies: 2024-04-10T22:40:29.843 INFO:teuthology.orchestra.run.smithi102.stdout: kernel-core x86_64 4.18.0-552.el8 baseos 44 M 2024-04-10T22:40:29.843 INFO:teuthology.orchestra.run.smithi102.stdout: kernel-modules x86_64 4.18.0-552.el8 baseos 36 M 2024-04-10T22:40:29.843 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T22:40:29.843 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction Summary 2024-04-10T22:40:29.843 INFO:teuthology.orchestra.run.smithi102.stdout:================================================================================ 2024-04-10T22:40:29.843 INFO:teuthology.orchestra.run.smithi102.stdout:Install 3 Packages 2024-04-10T22:40:29.843 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T22:40:29.844 INFO:teuthology.orchestra.run.smithi102.stdout:Total download size: 90 M 2024-04-10T22:40:29.844 INFO:teuthology.orchestra.run.smithi102.stdout:Installed size: 96 M 2024-04-10T22:40:29.844 INFO:teuthology.orchestra.run.smithi102.stdout:Downloading Packages: 2024-04-10T22:40:33.020 INFO:teuthology.orchestra.run.smithi102.stdout:(1/3): kernel-4.18.0-552.el8.x86_64.rpm 4.1 MB/s | 10 MB 00:02 2024-04-10T22:40:43.669 INFO:teuthology.orchestra.run.smithi102.stdout:(2/3): kernel-modules-4.18.0-552.el8.x86_64.rpm 2.8 MB/s | 36 MB 00:13 2024-04-10T22:40:50.470 INFO:teuthology.orchestra.run.smithi102.stdout:(3/3): kernel-core-4.18.0-552.el8.x86_64.rpm 2.2 MB/s | 44 MB 00:20 2024-04-10T22:40:50.470 INFO:teuthology.orchestra.run.smithi102.stdout:-------------------------------------------------------------------------------- 2024-04-10T22:40:50.470 INFO:teuthology.orchestra.run.smithi102.stdout:Total 4.4 MB/s | 90 MB 00:20 2024-04-10T22:40:51.155 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction check 2024-04-10T22:40:51.580 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction check succeeded. 2024-04-10T22:40:51.581 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction test 2024-04-10T22:40:52.829 INFO:teuthology.orchestra.run.smithi102.stdout:Transaction test succeeded. 2024-04-10T22:40:52.861 INFO:teuthology.orchestra.run.smithi102.stdout:Running transaction 2024-04-10T22:40:55.657 INFO:teuthology.orchestra.run.smithi102.stdout: Preparing : 1/1 2024-04-10T22:40:56.253 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:40:59.874 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:41:00.286 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:41:03.735 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:41:04.034 INFO:teuthology.orchestra.run.smithi102.stdout: Installing : kernel-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:41:26.040 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:41:26.064 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:41:26.415 INFO:teuthology.orchestra.run.smithi102.stdout: Running scriptlet: kernel-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:41:26.416 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : kernel-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:41:26.416 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : kernel-core-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:41:26.944 INFO:teuthology.orchestra.run.smithi102.stdout: Verifying : kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:41:26.945 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T22:41:26.945 INFO:teuthology.orchestra.run.smithi102.stdout:Installed: 2024-04-10T22:41:26.945 INFO:teuthology.orchestra.run.smithi102.stdout: kernel-4.18.0-552.el8.x86_64 kernel-core-4.18.0-552.el8.x86_64 2024-04-10T22:41:26.945 INFO:teuthology.orchestra.run.smithi102.stdout: kernel-modules-4.18.0-552.el8.x86_64 2024-04-10T22:41:26.945 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T22:41:26.945 INFO:teuthology.orchestra.run.smithi102.stdout:Complete! 2024-04-10T22:41:27.092 DEBUG:teuthology.orchestra.run.smithi102:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:41:27.307 INFO:teuthology.orchestra.run.smithi102.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-10T22:41:27.308 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-10T22:41:27.308 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-552.el8.x86_64 2024-04-10T22:41:27.308 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-10T22:41:27.308 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-10T22:41:27.308 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-04-10T22:41:27.309 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi102.front.sepia.ceph.com, path=None, version=distro) 2024-04-10T22:41:27.309 DEBUG:teuthology.orchestra.run.smithi102:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:41:27.519 INFO:teuthology.orchestra.run.smithi102.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-10T22:41:27.520 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-10T22:41:27.520 DEBUG:teuthology.orchestra.run.smithi102:> sudo rpm -qi grub2-tools 2024-04-10T22:41:27.553 INFO:teuthology.orchestra.run.smithi102.stdout:Name : grub2-tools 2024-04-10T22:41:27.554 INFO:teuthology.orchestra.run.smithi102.stdout:Epoch : 1 2024-04-10T22:41:27.554 INFO:teuthology.orchestra.run.smithi102.stdout:Version : 2.02 2024-04-10T22:41:27.554 INFO:teuthology.orchestra.run.smithi102.stdout:Release : 129.el8 2024-04-10T22:41:27.554 INFO:teuthology.orchestra.run.smithi102.stdout:Architecture: x86_64 2024-04-10T22:41:27.554 INFO:teuthology.orchestra.run.smithi102.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2024-04-10T22:41:27.554 INFO:teuthology.orchestra.run.smithi102.stdout:Group : System Environment/Base 2024-04-10T22:41:27.554 INFO:teuthology.orchestra.run.smithi102.stdout:Size : 9265586 2024-04-10T22:41:27.554 INFO:teuthology.orchestra.run.smithi102.stdout:License : GPLv3+ 2024-04-10T22:41:27.554 INFO:teuthology.orchestra.run.smithi102.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2024-04-10T22:41:27.554 INFO:teuthology.orchestra.run.smithi102.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2024-04-10T22:41:27.555 INFO:teuthology.orchestra.run.smithi102.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2024-04-10T22:41:27.555 INFO:teuthology.orchestra.run.smithi102.stdout:Build Host : kbuilder.bsys.centos.org 2024-04-10T22:41:27.555 INFO:teuthology.orchestra.run.smithi102.stdout:Relocations : (not relocatable) 2024-04-10T22:41:27.555 INFO:teuthology.orchestra.run.smithi102.stdout:Packager : CentOS BuildSystem 2024-04-10T22:41:27.555 INFO:teuthology.orchestra.run.smithi102.stdout:Vendor : CentOS 2024-04-10T22:41:27.555 INFO:teuthology.orchestra.run.smithi102.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-10T22:41:27.555 INFO:teuthology.orchestra.run.smithi102.stdout:Summary : Support tools for GRUB. 2024-04-10T22:41:27.555 INFO:teuthology.orchestra.run.smithi102.stdout:Description : 2024-04-10T22:41:27.555 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T22:41:27.555 INFO:teuthology.orchestra.run.smithi102.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-10T22:41:27.555 INFO:teuthology.orchestra.run.smithi102.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-10T22:41:27.555 INFO:teuthology.orchestra.run.smithi102.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-10T22:41:27.555 INFO:teuthology.orchestra.run.smithi102.stdout:hardware devices. 2024-04-10T22:41:27.556 INFO:teuthology.orchestra.run.smithi102.stdout: 2024-04-10T22:41:27.556 INFO:teuthology.orchestra.run.smithi102.stdout:This subpackage provides tools for support of all platforms. 2024-04-10T22:41:27.556 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-10T22:41:27.557 INFO:teuthology.task.kernel:Updating grub on smithi102 to boot 4.18.0-552.el8.x86_64 2024-04-10T22:41:27.557 DEBUG:teuthology.orchestra.run.smithi102:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-10T22:41:28.987 INFO:teuthology.orchestra.run.smithi102.stderr:Generating grub configuration file ... 2024-04-10T22:41:31.279 INFO:teuthology.orchestra.run.smithi102.stderr:done 2024-04-10T22:41:31.281 DEBUG:teuthology.orchestra.run.smithi102:> mktemp 2024-04-10T22:41:31.315 INFO:teuthology.orchestra.run.smithi102.stdout:/tmp/tmp.BwLKZwKQnY 2024-04-10T22:41:31.315 DEBUG:teuthology.orchestra.run.smithi102:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.BwLKZwKQnY 2024-04-10T22:41:31.389 DEBUG:teuthology.orchestra.run.smithi102:> sudo chmod 0666 /tmp/tmp.BwLKZwKQnY 2024-04-10T22:41:31.543 DEBUG:teuthology.orchestra.remote:smithi102:/tmp/tmp.BwLKZwKQnY is 6KB 2024-04-10T22:41:31.592 DEBUG:teuthology.orchestra.run.smithi102:> rm -fr /tmp/tmp.BwLKZwKQnY 2024-04-10T22:41:31.609 DEBUG:teuthology.orchestra.run.smithi102:> sudo /bin/ls /boot/loader/entries || true 2024-04-10T22:41:31.693 INFO:teuthology.orchestra.run.smithi102.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2024-04-10T22:41:31.693 INFO:teuthology.orchestra.run.smithi102.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2024-04-10T22:41:31.693 INFO:teuthology.orchestra.run.smithi102.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2024-04-10T22:41:31.693 INFO:teuthology.orchestra.run.smithi102.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64.conf 2024-04-10T22:41:31.694 DEBUG:teuthology.orchestra.run.smithi102:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64 2024-04-10T22:41:31.815 DEBUG:teuthology.orchestra.run.smithi102:> sudo shutdown -r now 2024-04-10T22:41:46.638 INFO:teuthology.orchestra.run.smithi138.stdout:CentOS Stream 8 - Extras common packages 60 B/s | 7.7 kB 02:12 2024-04-10T22:42:01.818 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-04-10T22:42:01.819 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi102.front.sepia.ceph.com' 2024-04-10T22:42:01.820 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi102.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:42:20.318 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.102 2024-04-10T22:42:27.019 INFO:teuthology.orchestra.run.smithi138.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:40 2024-04-10T22:42:27.019 INFO:teuthology.orchestra.run.smithi138.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:42:27.020 INFO:teuthology.orchestra.run.smithi138.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:42:27.020 INFO:teuthology.orchestra.run.smithi138.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:42:27.985 INFO:teuthology.orchestra.run.smithi138.stdout:Extra Packages for Enterprise Linux 17 MB/s | 16 MB 00:00 2024-04-10T22:42:29.319 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi102.front.sepia.ceph.com' 2024-04-10T22:42:29.320 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi102.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:42:31.718 INFO:teuthology.orchestra.run.smithi138.stdout:lab-extras 450 kB/s | 24 kB 00:00 2024-04-10T22:42:31.868 INFO:teuthology.orchestra.run.smithi138.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:42:32.382 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.102 2024-04-10T22:42:34.774 INFO:teuthology.orchestra.run.smithi138.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2024-04-10T22:42:34.774 INFO:teuthology.orchestra.run.smithi138.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2024-04-10T22:42:34.884 INFO:teuthology.orchestra.run.smithi138.stdout:Dependencies resolved. 2024-04-10T22:42:34.885 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2024-04-10T22:42:34.885 INFO:teuthology.orchestra.run.smithi138.stdout: Package Architecture Version Repository Size 2024-04-10T22:42:34.885 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2024-04-10T22:42:34.886 INFO:teuthology.orchestra.run.smithi138.stdout:Installing: 2024-04-10T22:42:34.886 INFO:teuthology.orchestra.run.smithi138.stdout: kernel x86_64 4.18.0-552.el8 baseos 10 M 2024-04-10T22:42:34.886 INFO:teuthology.orchestra.run.smithi138.stdout:Installing dependencies: 2024-04-10T22:42:34.886 INFO:teuthology.orchestra.run.smithi138.stdout: kernel-core x86_64 4.18.0-552.el8 baseos 44 M 2024-04-10T22:42:34.886 INFO:teuthology.orchestra.run.smithi138.stdout: kernel-modules x86_64 4.18.0-552.el8 baseos 36 M 2024-04-10T22:42:34.886 INFO:teuthology.orchestra.run.smithi138.stdout: 2024-04-10T22:42:34.886 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction Summary 2024-04-10T22:42:34.886 INFO:teuthology.orchestra.run.smithi138.stdout:================================================================================ 2024-04-10T22:42:34.886 INFO:teuthology.orchestra.run.smithi138.stdout:Install 3 Packages 2024-04-10T22:42:34.886 INFO:teuthology.orchestra.run.smithi138.stdout: 2024-04-10T22:42:34.886 INFO:teuthology.orchestra.run.smithi138.stdout:Total download size: 90 M 2024-04-10T22:42:34.886 INFO:teuthology.orchestra.run.smithi138.stdout:Installed size: 96 M 2024-04-10T22:42:34.886 INFO:teuthology.orchestra.run.smithi138.stdout:Downloading Packages: 2024-04-10T22:42:44.383 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi102.front.sepia.ceph.com' 2024-04-10T22:42:44.384 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi102.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:43:02.818 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.102 2024-04-10T22:43:17.820 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi102.front.sepia.ceph.com' 2024-04-10T22:43:17.821 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi102.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:43:18.891 DEBUG:teuthology.orchestra.run.smithi102:> true 2024-04-10T22:43:19.387 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi102.front.sepia.ceph.com' 2024-04-10T22:43:19.388 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "4.18.0-552.el8.x86_64"... 2024-04-10T22:43:19.388 DEBUG:teuthology.orchestra.run.smithi102:> uname -r 2024-04-10T22:43:19.406 INFO:teuthology.orchestra.run.smithi102.stdout:4.18.0-552.el8.x86_64 2024-04-10T22:43:19.406 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-552.el8.x86_64 vs 4.18.0-552.el8.x86_64 2024-04-10T22:43:19.406 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-10T22:43:19.406 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-10T22:43:20.407 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-04-10T22:43:20.408 DEBUG:teuthology.orchestra.run.smithi102:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-10T22:43:20.468 INFO:teuthology.orchestra.run.smithi102.stdout:ttyS1 2024-04-10T22:43:20.480 DEBUG:teuthology.parallel:result is None 2024-04-10T22:44:57.517 INFO:teuthology.orchestra.run.smithi138.stdout:(1/3): kernel-4.18.0-552.el8.x86_64.rpm 1.0 MB/s | 10 MB 00:10 2024-04-10T22:45:13.190 INFO:teuthology.orchestra.run.smithi138.stdout:(2/3): kernel-core-4.18.0-552.el8.x86_64.rpm 1.7 MB/s | 44 MB 00:26 2024-04-10T22:45:14.140 INFO:teuthology.orchestra.run.smithi138.stdout:(3/3): kernel-modules-4.18.0-552.el8.x86_64.rpm 1.3 MB/s | 36 MB 00:27 2024-04-10T22:45:14.140 INFO:teuthology.orchestra.run.smithi138.stdout:-------------------------------------------------------------------------------- 2024-04-10T22:45:14.140 INFO:teuthology.orchestra.run.smithi138.stdout:Total 582 kB/s | 90 MB 02:39 2024-04-10T22:45:14.826 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction check 2024-04-10T22:45:15.265 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction check succeeded. 2024-04-10T22:45:15.265 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction test 2024-04-10T22:45:16.509 INFO:teuthology.orchestra.run.smithi138.stdout:Transaction test succeeded. 2024-04-10T22:45:16.545 INFO:teuthology.orchestra.run.smithi138.stdout:Running transaction 2024-04-10T22:45:19.370 INFO:teuthology.orchestra.run.smithi138.stdout: Preparing : 1/1 2024-04-10T22:45:19.934 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:45:23.562 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:45:23.959 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:45:27.394 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:45:27.699 INFO:teuthology.orchestra.run.smithi138.stdout: Installing : kernel-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:45:49.924 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:45:49.947 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:45:50.439 INFO:teuthology.orchestra.run.smithi138.stdout: Running scriptlet: kernel-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:45:50.439 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : kernel-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:45:50.439 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : kernel-core-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:45:50.975 INFO:teuthology.orchestra.run.smithi138.stdout: Verifying : kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:45:50.975 INFO:teuthology.orchestra.run.smithi138.stdout: 2024-04-10T22:45:50.975 INFO:teuthology.orchestra.run.smithi138.stdout:Installed: 2024-04-10T22:45:50.975 INFO:teuthology.orchestra.run.smithi138.stdout: kernel-4.18.0-552.el8.x86_64 kernel-core-4.18.0-552.el8.x86_64 2024-04-10T22:45:50.975 INFO:teuthology.orchestra.run.smithi138.stdout: kernel-modules-4.18.0-552.el8.x86_64 2024-04-10T22:45:50.975 INFO:teuthology.orchestra.run.smithi138.stdout: 2024-04-10T22:45:50.975 INFO:teuthology.orchestra.run.smithi138.stdout:Complete! 2024-04-10T22:45:51.123 DEBUG:teuthology.orchestra.run.smithi138:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:45:51.342 INFO:teuthology.orchestra.run.smithi138.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-10T22:45:51.343 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-10T22:45:51.343 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-552.el8.x86_64 2024-04-10T22:45:51.343 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-10T22:45:51.343 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-10T22:45:51.343 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-04-10T22:45:51.343 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi138.front.sepia.ceph.com, path=None, version=distro) 2024-04-10T22:45:51.343 DEBUG:teuthology.orchestra.run.smithi138:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:45:51.556 INFO:teuthology.orchestra.run.smithi138.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-10T22:45:51.556 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-10T22:45:51.556 DEBUG:teuthology.orchestra.run.smithi138:> sudo rpm -qi grub2-tools 2024-04-10T22:45:51.590 INFO:teuthology.orchestra.run.smithi138.stdout:Name : grub2-tools 2024-04-10T22:45:51.590 INFO:teuthology.orchestra.run.smithi138.stdout:Epoch : 1 2024-04-10T22:45:51.590 INFO:teuthology.orchestra.run.smithi138.stdout:Version : 2.02 2024-04-10T22:45:51.590 INFO:teuthology.orchestra.run.smithi138.stdout:Release : 129.el8 2024-04-10T22:45:51.590 INFO:teuthology.orchestra.run.smithi138.stdout:Architecture: x86_64 2024-04-10T22:45:51.590 INFO:teuthology.orchestra.run.smithi138.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2024-04-10T22:45:51.590 INFO:teuthology.orchestra.run.smithi138.stdout:Group : System Environment/Base 2024-04-10T22:45:51.590 INFO:teuthology.orchestra.run.smithi138.stdout:Size : 9265586 2024-04-10T22:45:51.591 INFO:teuthology.orchestra.run.smithi138.stdout:License : GPLv3+ 2024-04-10T22:45:51.591 INFO:teuthology.orchestra.run.smithi138.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2024-04-10T22:45:51.591 INFO:teuthology.orchestra.run.smithi138.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2024-04-10T22:45:51.591 INFO:teuthology.orchestra.run.smithi138.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2024-04-10T22:45:51.591 INFO:teuthology.orchestra.run.smithi138.stdout:Build Host : kbuilder.bsys.centos.org 2024-04-10T22:45:51.591 INFO:teuthology.orchestra.run.smithi138.stdout:Relocations : (not relocatable) 2024-04-10T22:45:51.591 INFO:teuthology.orchestra.run.smithi138.stdout:Packager : CentOS BuildSystem 2024-04-10T22:45:51.591 INFO:teuthology.orchestra.run.smithi138.stdout:Vendor : CentOS 2024-04-10T22:45:51.591 INFO:teuthology.orchestra.run.smithi138.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-10T22:45:51.591 INFO:teuthology.orchestra.run.smithi138.stdout:Summary : Support tools for GRUB. 2024-04-10T22:45:51.591 INFO:teuthology.orchestra.run.smithi138.stdout:Description : 2024-04-10T22:45:51.591 INFO:teuthology.orchestra.run.smithi138.stdout: 2024-04-10T22:45:51.591 INFO:teuthology.orchestra.run.smithi138.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-10T22:45:51.591 INFO:teuthology.orchestra.run.smithi138.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-10T22:45:51.591 INFO:teuthology.orchestra.run.smithi138.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-10T22:45:51.592 INFO:teuthology.orchestra.run.smithi138.stdout:hardware devices. 2024-04-10T22:45:51.592 INFO:teuthology.orchestra.run.smithi138.stdout: 2024-04-10T22:45:51.592 INFO:teuthology.orchestra.run.smithi138.stdout:This subpackage provides tools for support of all platforms. 2024-04-10T22:45:51.592 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-10T22:45:51.593 INFO:teuthology.task.kernel:Updating grub on smithi138 to boot 4.18.0-552.el8.x86_64 2024-04-10T22:45:51.593 DEBUG:teuthology.orchestra.run.smithi138:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-10T22:45:53.085 INFO:teuthology.orchestra.run.smithi138.stderr:Generating grub configuration file ... 2024-04-10T22:45:55.042 INFO:teuthology.orchestra.run.smithi138.stderr:done 2024-04-10T22:45:55.045 DEBUG:teuthology.orchestra.run.smithi138:> mktemp 2024-04-10T22:45:55.074 INFO:teuthology.orchestra.run.smithi138.stdout:/tmp/tmp.woxP7qerGa 2024-04-10T22:45:55.074 DEBUG:teuthology.orchestra.run.smithi138:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.woxP7qerGa 2024-04-10T22:45:55.149 DEBUG:teuthology.orchestra.run.smithi138:> sudo chmod 0666 /tmp/tmp.woxP7qerGa 2024-04-10T22:45:55.310 DEBUG:teuthology.orchestra.remote:smithi138:/tmp/tmp.woxP7qerGa is 6KB 2024-04-10T22:45:55.358 DEBUG:teuthology.orchestra.run.smithi138:> rm -fr /tmp/tmp.woxP7qerGa 2024-04-10T22:45:55.375 DEBUG:teuthology.orchestra.run.smithi138:> sudo /bin/ls /boot/loader/entries || true 2024-04-10T22:45:55.455 INFO:teuthology.orchestra.run.smithi138.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2024-04-10T22:45:55.455 INFO:teuthology.orchestra.run.smithi138.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2024-04-10T22:45:55.455 INFO:teuthology.orchestra.run.smithi138.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2024-04-10T22:45:55.455 INFO:teuthology.orchestra.run.smithi138.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64.conf 2024-04-10T22:45:55.457 DEBUG:teuthology.orchestra.run.smithi138:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64 2024-04-10T22:45:55.659 DEBUG:teuthology.orchestra.run.smithi138:> sudo shutdown -r now 2024-04-10T22:46:25.661 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-04-10T22:46:25.662 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi138.front.sepia.ceph.com' 2024-04-10T22:46:25.663 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi138.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:47:00.383 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.138 2024-04-10T22:47:09.384 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi138.front.sepia.ceph.com' 2024-04-10T22:47:09.385 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi138.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:47:12.446 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.138 2024-04-10T22:47:24.449 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi138.front.sepia.ceph.com' 2024-04-10T22:47:24.450 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi138.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:47:56.443 DEBUG:teuthology.orchestra.run.smithi138:> true 2024-04-10T22:47:56.849 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi138.front.sepia.ceph.com' 2024-04-10T22:47:56.849 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "4.18.0-552.el8.x86_64"... 2024-04-10T22:47:56.849 DEBUG:teuthology.orchestra.run.smithi138:> uname -r 2024-04-10T22:47:56.865 INFO:teuthology.orchestra.run.smithi138.stdout:4.18.0-552.el8.x86_64 2024-04-10T22:47:56.866 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-552.el8.x86_64 vs 4.18.0-552.el8.x86_64 2024-04-10T22:47:56.866 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-10T22:47:56.866 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-10T22:47:57.867 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-04-10T22:47:57.868 DEBUG:teuthology.orchestra.run.smithi138:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-10T22:47:57.959 INFO:teuthology.orchestra.run.smithi138.stdout:ttyS1 2024-04-10T22:47:57.972 DEBUG:teuthology.parallel:result is None 2024-04-10T22:47:57.973 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-10T22:47:57.981 INFO:teuthology.task.internal:Creating test directory... 2024-04-10T22:47:57.981 DEBUG:teuthology.orchestra.run.smithi102:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-10T22:47:57.984 DEBUG:teuthology.orchestra.run.smithi138:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-10T22:47:58.004 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-10T22:47:58.012 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-10T22:47:58.021 INFO:teuthology.task.internal:Creating archive directory... 2024-04-10T22:47:58.021 DEBUG:teuthology.orchestra.run.smithi102:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-10T22:47:58.047 DEBUG:teuthology.orchestra.run.smithi138:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-10T22:47:58.079 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-10T22:47:58.087 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-10T22:47:58.087 DEBUG:teuthology.orchestra.run.smithi102:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-10T22:47:58.116 DEBUG:teuthology.orchestra.run.smithi138:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-10T22:47:58.152 INFO:teuthology.orchestra.run.smithi138.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:47:58.152 INFO:teuthology.orchestra.run.smithi102.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:47:58.162 INFO:teuthology.orchestra.run.smithi138.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:47:58.164 INFO:teuthology.orchestra.run.smithi102.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:47:58.166 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-10T22:47:58.173 INFO:teuthology.task.internal:Configuring sudo... 2024-04-10T22:47:58.174 DEBUG:teuthology.orchestra.run.smithi102:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-10T22:47:58.209 DEBUG:teuthology.orchestra.run.smithi138:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-10T22:47:58.239 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-10T22:47:58.251 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-10T22:47:58.252 DEBUG:teuthology.orchestra.run.smithi102:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-10T22:47:58.282 DEBUG:teuthology.orchestra.run.smithi138:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-10T22:47:58.301 DEBUG:teuthology.orchestra.run.smithi102:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:47:58.391 DEBUG:teuthology.orchestra.run.smithi102:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:47:58.448 DEBUG:teuthology.orchestra.run.smithi102:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:47:58.535 DEBUG:teuthology.orchestra.run.smithi102:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:47:58.585 DEBUG:teuthology.orchestra.run.smithi102:> set -ex 2024-04-10T22:47:58.585 DEBUG:teuthology.orchestra.run.smithi102:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-10T22:47:58.658 DEBUG:teuthology.orchestra.run.smithi138:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:47:58.702 DEBUG:teuthology.orchestra.run.smithi138:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:47:58.766 DEBUG:teuthology.orchestra.run.smithi138:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:47:58.809 DEBUG:teuthology.orchestra.run.smithi138:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:47:58.851 DEBUG:teuthology.orchestra.run.smithi138:> set -ex 2024-04-10T22:47:58.852 DEBUG:teuthology.orchestra.run.smithi138:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-10T22:47:58.920 DEBUG:teuthology.orchestra.run.smithi102:> sudo service rsyslog restart 2024-04-10T22:47:58.922 DEBUG:teuthology.orchestra.run.smithi138:> sudo service rsyslog restart 2024-04-10T22:47:58.992 INFO:teuthology.orchestra.run.smithi102.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:47:59.016 INFO:teuthology.orchestra.run.smithi138.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:47:59.429 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-10T22:47:59.440 INFO:teuthology.task.internal:Starting timer... 2024-04-10T22:47:59.440 INFO:teuthology.run_tasks:Running task pcp... 2024-04-10T22:47:59.481 INFO:teuthology.run_tasks:Running task selinux... 2024-04-10T22:47:59.499 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-04-10T22:47:59.499 DEBUG:teuthology.orchestra.run.smithi102:> sudo service auditd rotate 2024-04-10T22:47:59.589 INFO:teuthology.orchestra.run.smithi102.stdout:Rotating logs: 2024-04-10T22:47:59.591 DEBUG:teuthology.orchestra.run.smithi138:> sudo service auditd rotate 2024-04-10T22:47:59.721 INFO:teuthology.orchestra.run.smithi138.stdout:Rotating logs: 2024-04-10T22:47:59.723 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-10T22:47:59.723 DEBUG:teuthology.orchestra.run.smithi102:> /usr/sbin/getenforce 2024-04-10T22:47:59.751 INFO:teuthology.orchestra.run.smithi102.stdout:Permissive 2024-04-10T22:47:59.751 DEBUG:teuthology.orchestra.run.smithi138:> /usr/sbin/getenforce 2024-04-10T22:47:59.782 INFO:teuthology.orchestra.run.smithi138.stdout:Permissive 2024-04-10T22:47:59.782 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi102.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi138.front.sepia.ceph.com': 'permissive'} 2024-04-10T22:47:59.782 DEBUG:teuthology.orchestra.run.smithi102:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:47:59.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:47:59.820 DEBUG:teuthology.orchestra.run.smithi138:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:47:59.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:47:59.847 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-10T22:47:59.847 DEBUG:teuthology.orchestra.run.smithi102:> sudo /usr/sbin/setenforce permissive 2024-04-10T22:47:59.888 DEBUG:teuthology.orchestra.run.smithi138:> sudo /usr/sbin/setenforce permissive 2024-04-10T22:47:59.975 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-10T22:47:59.985 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-04-10T22:47:59.991 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-04-10T22:48:00.120 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-10T22:48:00.138 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-10T22:48:00.139 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi102.front.sepia.ceph.com,smithi138.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-10T23:03:09.705 INFO:teuthology.task.ansible:Archiving ansible failure log at: /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650194/ansible_failures.yaml 2024-04-10T23:03:09.707 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/__init__.py", line 123, in __enter__ self.begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 508, in begin super(CephLab, self).begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 342, in begin self.execute_playbook() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 374, in execute_playbook self._handle_failure(command, status) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 402, in _handle_failure raise AnsibleFailedError(failures) teuthology.exceptions.AnsibleFailedError: [',', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at'] 2024-04-10T23:03:09.939 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=39367293451944e7bc47148a8d63c595 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/__init__.py", line 123, in __enter__ self.begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 508, in begin super(CephLab, self).begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 342, in begin self.execute_playbook() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 374, in execute_playbook self._handle_failure(command, status) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 402, in _handle_failure raise AnsibleFailedError(failures) teuthology.exceptions.AnsibleFailedError: [',', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at'] 2024-04-10T23:03:09.941 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-10T23:03:09.950 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-10T23:03:09.950 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-10T23:03:09.965 DEBUG:teuthology.orchestra.run.smithi102:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-10T23:03:10.052 DEBUG:teuthology.orchestra.run.smithi138:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-10T23:03:10.121 DEBUG:teuthology.orchestra.run.smithi102:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T23:03:10.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T23:03:10.153 DEBUG:teuthology.orchestra.run.smithi138:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T23:03:10.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T23:03:10.184 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-10T23:03:10.193 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-10T23:03:10.201 INFO:teuthology.task.internal:Duration was 910.760561 seconds 2024-04-10T23:03:10.201 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-10T23:03:10.209 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-10T23:03:10.209 DEBUG:teuthology.orchestra.run.smithi102:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-10T23:03:10.212 DEBUG:teuthology.orchestra.run.smithi138:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-10T23:03:10.258 INFO:teuthology.orchestra.run.smithi102.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T23:03:10.273 INFO:teuthology.orchestra.run.smithi138.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T23:03:10.426 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-10T23:03:10.427 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi102.front.sepia.ceph.com 2024-04-10T23:03:10.427 DEBUG:teuthology.orchestra.run.smithi102:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-10T23:03:10.450 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi138.front.sepia.ceph.com 2024-04-10T23:03:10.450 DEBUG:teuthology.orchestra.run.smithi138:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-10T23:03:10.472 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-10T23:03:10.472 DEBUG:teuthology.orchestra.run.smithi102:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-10T23:03:10.493 DEBUG:teuthology.orchestra.run.smithi138:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-10T23:03:10.551 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-10T23:03:10.553 DEBUG:teuthology.orchestra.run.smithi102:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-10T23:03:10.574 DEBUG:teuthology.orchestra.run.smithi138:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-10T23:03:10.652 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-10T23:03:10.662 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-10T23:03:10.662 DEBUG:teuthology.orchestra.run.smithi102:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-10T23:03:10.689 DEBUG:teuthology.orchestra.run.smithi138:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-10T23:03:10.715 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-10T23:03:10.724 DEBUG:teuthology.orchestra.run.smithi102:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-10T23:03:10.732 DEBUG:teuthology.orchestra.run.smithi138:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-10T23:03:10.756 INFO:teuthology.orchestra.run.smithi102.stdout:kernel.core_pattern = core 2024-04-10T23:03:10.781 INFO:teuthology.orchestra.run.smithi138.stdout:kernel.core_pattern = core 2024-04-10T23:03:10.815 DEBUG:teuthology.orchestra.run.smithi102:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-10T23:03:10.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T23:03:10.837 DEBUG:teuthology.orchestra.run.smithi138:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-10T23:03:10.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T23:03:10.872 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-10T23:03:10.881 INFO:teuthology.task.internal:Transferring archived files... 2024-04-10T23:03:10.882 DEBUG:teuthology.misc:Transferring archived files from smithi102:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650194/remote/smithi102 2024-04-10T23:03:10.884 DEBUG:teuthology.orchestra.run.smithi102:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-10T23:03:10.939 DEBUG:teuthology.misc:Transferring archived files from smithi138:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650194/remote/smithi138 2024-04-10T23:03:10.940 DEBUG:teuthology.orchestra.run.smithi138:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-10T23:03:10.986 INFO:teuthology.task.internal:Removing archive directory... 2024-04-10T23:03:10.987 DEBUG:teuthology.orchestra.run.smithi102:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-10T23:03:10.989 DEBUG:teuthology.orchestra.run.smithi138:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-10T23:03:11.038 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-10T23:03:11.046 INFO:teuthology.task.internal:Not uploading archives. 2024-04-10T23:03:11.047 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-10T23:03:11.055 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-10T23:03:11.055 DEBUG:teuthology.orchestra.run.smithi102:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-10T23:03:11.057 DEBUG:teuthology.orchestra.run.smithi138:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-10T23:03:11.072 INFO:teuthology.orchestra.run.smithi102.stdout: 393224 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 10 23:03 /home/ubuntu/cephtest 2024-04-10T23:03:11.095 INFO:teuthology.orchestra.run.smithi138.stdout: 525401 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 10 23:03 /home/ubuntu/cephtest 2024-04-10T23:03:11.096 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-10T23:03:11.105 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-10T23:03:11.124 INFO:teuthology.nuke:Checking targets against current locks 2024-04-10T23:03:11.164 DEBUG:teuthology.nuke:shortname: smithi102 2024-04-10T23:03:11.165 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T23:03:11.249 DEBUG:teuthology.nuke:shortname: smithi138 2024-04-10T23:03:11.249 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T23:03:11.264 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi102.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650194', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:21.306042', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAYn7c2irT2HVTr9SoxIe4TY/qZoOOBNNp+2Avooi8GZGS39lhLelR8BlBGukMHHBw0wp/uClqWiy6fjqKVuyTY='} 2024-04-10T23:03:11.299 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi138.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650194', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:21.307848', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNI9I97+Z3WbG7lfx7sIKyYunYVgYJqMabmJeNdERfodMr6K9VDGoAm7Usm3pZlIa/DCRqG5FIepX+SoaUNacwQ='} 2024-04-10T23:03:11.381 INFO:teuthology.orchestra.console.smithi102:Power off 2024-04-10T23:03:11.381 DEBUG:teuthology.orchestra.console.smithi102:pexpect command: ipmitool -H smithi102.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-10T23:03:11.421 INFO:teuthology.orchestra.console.smithi138:Power off 2024-04-10T23:03:11.421 DEBUG:teuthology.orchestra.console.smithi138:pexpect command: ipmitool -H smithi138.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-10T23:03:11.456 DEBUG:teuthology.orchestra.console.smithi102:power off output: Chassis Power Control: Down/Off 2024-04-10T23:03:11.456 DEBUG:teuthology.orchestra.console.smithi102:pexpect command: ipmitool -H smithi102.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:03:11.474 DEBUG:teuthology.orchestra.console.smithi138:power off output: Chassis Power Control: Down/Off 2024-04-10T23:03:11.474 DEBUG:teuthology.orchestra.console.smithi138:pexpect command: ipmitool -H smithi138.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:03:11.516 DEBUG:teuthology.orchestra.console.smithi102:check power output: Chassis Power is on 2024-04-10T23:03:11.527 DEBUG:teuthology.orchestra.console.smithi138:check power output: Chassis Power is on 2024-04-10T23:03:15.518 DEBUG:teuthology.orchestra.console.smithi102:pexpect command: ipmitool -H smithi102.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:03:15.528 DEBUG:teuthology.orchestra.console.smithi138:pexpect command: ipmitool -H smithi138.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:03:15.629 DEBUG:teuthology.orchestra.console.smithi102:check power output: Chassis Power is on 2024-04-10T23:03:15.637 DEBUG:teuthology.orchestra.console.smithi138:check power output: Chassis Power is on 2024-04-10T23:03:19.634 DEBUG:teuthology.orchestra.console.smithi102:pexpect command: ipmitool -H smithi102.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:03:19.643 DEBUG:teuthology.orchestra.console.smithi138:pexpect command: ipmitool -H smithi138.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:03:19.745 DEBUG:teuthology.orchestra.console.smithi102:check power output: Chassis Power is on 2024-04-10T23:03:19.752 DEBUG:teuthology.orchestra.console.smithi138:check power output: Chassis Power is on 2024-04-10T23:03:23.750 DEBUG:teuthology.orchestra.console.smithi102:pexpect command: ipmitool -H smithi102.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:03:23.759 DEBUG:teuthology.orchestra.console.smithi138:pexpect command: ipmitool -H smithi138.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:03:23.860 DEBUG:teuthology.orchestra.console.smithi102:check power output: Chassis Power is off 2024-04-10T23:03:23.868 DEBUG:teuthology.orchestra.console.smithi138:check power output: Chassis Power is off 2024-04-10T23:03:23.962 INFO:teuthology.orchestra.console.smithi102:Power off completed 2024-04-10T23:03:23.969 INFO:teuthology.orchestra.console.smithi138:Power off completed 2024-04-10T23:03:24.075 INFO:teuthology.run:Summary data: description: upgrade:pacific-x/stress-split/{0-distro/centos_8.stream_container_tools_crun 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} duration: 910.7605612277985 failure_reason: '['','', ''Bugzilla. In case of problems, contact the owner of this repository.'', ''Enabling a Copr repository. Please note that this repository is not part'', ''Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable'', ''Please do not file bug reports about these packages in Fedora'', ''The Fedora Project does not exercise any power over the contents of'', ''and packages are not held to any quality or security level.'', ''of the main distribution, and quality may vary.'', ''this repository beyond the rules outlined in the Copr FAQ at'']' owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=39367293451944e7bc47148a8d63c595 status: dead success: false 2024-04-10T23:03:24.075 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-10T23:03:24.170 INFO:teuthology.run:DEAD