2024-04-10T22:38:33.806 INFO:root:teuthology version: 0.0.1.dev267+g6c63784 2024-04-10T22:38:33.806 DEBUG:teuthology.run:Teuthology command: teuthology --owner scheduled_teuthology@teuthology --verbose --name teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi --description upgrade:pacific-x/stress-split/{0-distro/centos_8.stream_container_tools_crun 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} --archive /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650174 -- /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650174/orig.config.yaml 2024-04-10T22:38:33.861 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-10T22:38:33.957 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650174 branch: reef description: upgrade:pacific-x/stress-split/{0-distro/centos_8.stream_container_tools_crun 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} email: ceph-qa@ceph.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' first_in_suite: false job_id: '7650174' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 8.stream overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: reef sha1: 4f70a9a90881107996c8b7a3e89ba8b32624713c owner: scheduled_teuthology@teuthology priority: 830 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 4604 sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: pacific clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 0/1 suite: upgrade:pacific-x suite_branch: reef suite_path: /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 4f70a9a90881107996c8b7a3e89ba8b32624713c targets: smithi027.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOznJROncvZ/sl4kvt1uMFmXbcvtyJA1c/fZSZQ8Sg2B4kUlYYsaE/1Z4ivcUVXCNmusjHqAExDZ5P5U2csuf4k= smithi044.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPxi/gvs1PpXnaMk8aKlR0U7WUu8169ktzHlQf7DjusykuABHNXGrpmKRK+S2aTz6JCmi8Mu1OE/mIAfT3G5MlM= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - sudo sed -i 's/runtime = "runc"/#runtime = "runc"/g' /usr/share/containers/containers.conf - sudo sed -i 's/#runtime = "crun"/runtime = "crun"/g' /usr/share/containers/containers.conf - install: branch: pacific exclude_packages: - ceph-volume - cephadm: cephadm_branch: pacific cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:pacific - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 6c637841c215537a4502385240412f1966e0faab timestamp: 2024-04-10_01:08:08 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-04-10T22:38:33.957 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa; will attempt to use it 2024-04-10T22:38:33.958 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa/tasks 2024-04-10T22:38:33.958 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-10T22:38:33.959 INFO:teuthology.task.internal:Checking packages... 2024-04-10T22:38:33.980 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'd540ebaca6b131a1dd560e7f69e024b133bbaa42' 2024-04-10T22:38:33.980 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-10T22:38:33.981 INFO:teuthology.packaging:ref: None 2024-04-10T22:38:33.981 INFO:teuthology.packaging:tag: None 2024-04-10T22:38:33.981 INFO:teuthology.packaging:branch: reef 2024-04-10T22:38:33.981 INFO:teuthology.packaging:sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 2024-04-10T22:38:33.981 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef 2024-04-10T22:38:34.360 INFO:teuthology.task.internal:Found packages for ceph version 18.2.2-1189.gc96c8d47 2024-04-10T22:38:34.362 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-10T22:38:34.369 INFO:teuthology.task.internal:no buildpackages task found 2024-04-10T22:38:34.369 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-10T22:38:34.384 INFO:teuthology.task.internal:Saving configuration 2024-04-10T22:38:34.405 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-10T22:38:34.411 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T22:38:34.435 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi027.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650174', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:13.075234', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOznJROncvZ/sl4kvt1uMFmXbcvtyJA1c/fZSZQ8Sg2B4kUlYYsaE/1Z4ivcUVXCNmusjHqAExDZ5P5U2csuf4k='} 2024-04-10T22:38:34.458 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi044.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650174', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:13.073913', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPxi/gvs1PpXnaMk8aKlR0U7WUu8169ktzHlQf7DjusykuABHNXGrpmKRK+S2aTz6JCmi8Mu1OE/mIAfT3G5MlM='} 2024-04-10T22:38:34.458 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-10T22:38:34.465 INFO:teuthology.task.internal:roles: ubuntu@smithi027.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-04-10T22:38:34.465 INFO:teuthology.task.internal:roles: ubuntu@smithi044.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-04-10T22:38:34.465 INFO:teuthology.run_tasks:Running task console_log... 2024-04-10T22:38:34.530 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f21de7090d0>, signals=[15]) 2024-04-10T22:38:34.530 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-10T22:38:34.536 INFO:teuthology.task.internal:Opening connections... 2024-04-10T22:38:34.536 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi027.front.sepia.ceph.com 2024-04-10T22:38:34.538 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi027.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:38:34.612 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi044.front.sepia.ceph.com 2024-04-10T22:38:34.613 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi044.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:38:34.689 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-10T22:38:34.697 DEBUG:teuthology.orchestra.run.smithi027:> uname -m 2024-04-10T22:38:34.721 INFO:teuthology.orchestra.run.smithi027.stdout:x86_64 2024-04-10T22:38:34.721 DEBUG:teuthology.orchestra.run.smithi027:> cat /etc/os-release 2024-04-10T22:38:34.782 INFO:teuthology.orchestra.run.smithi027.stdout:NAME="CentOS Stream" 2024-04-10T22:38:34.782 INFO:teuthology.orchestra.run.smithi027.stdout:VERSION="8" 2024-04-10T22:38:34.782 INFO:teuthology.orchestra.run.smithi027.stdout:ID="centos" 2024-04-10T22:38:34.782 INFO:teuthology.orchestra.run.smithi027.stdout:ID_LIKE="rhel fedora" 2024-04-10T22:38:34.782 INFO:teuthology.orchestra.run.smithi027.stdout:VERSION_ID="8" 2024-04-10T22:38:34.783 INFO:teuthology.orchestra.run.smithi027.stdout:PLATFORM_ID="platform:el8" 2024-04-10T22:38:34.783 INFO:teuthology.orchestra.run.smithi027.stdout:PRETTY_NAME="CentOS Stream 8" 2024-04-10T22:38:34.783 INFO:teuthology.orchestra.run.smithi027.stdout:ANSI_COLOR="0;31" 2024-04-10T22:38:34.783 INFO:teuthology.orchestra.run.smithi027.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2024-04-10T22:38:34.783 INFO:teuthology.orchestra.run.smithi027.stdout:HOME_URL="https://centos.org/" 2024-04-10T22:38:34.783 INFO:teuthology.orchestra.run.smithi027.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-10T22:38:34.783 INFO:teuthology.orchestra.run.smithi027.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2024-04-10T22:38:34.783 INFO:teuthology.orchestra.run.smithi027.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-10T22:38:34.784 INFO:teuthology.lock.ops:Updating smithi027.front.sepia.ceph.com on lock server 2024-04-10T22:38:34.810 DEBUG:teuthology.orchestra.run.smithi044:> uname -m 2024-04-10T22:38:34.833 INFO:teuthology.orchestra.run.smithi044.stdout:x86_64 2024-04-10T22:38:34.833 DEBUG:teuthology.orchestra.run.smithi044:> cat /etc/os-release 2024-04-10T22:38:34.895 INFO:teuthology.orchestra.run.smithi044.stdout:NAME="CentOS Stream" 2024-04-10T22:38:34.895 INFO:teuthology.orchestra.run.smithi044.stdout:VERSION="8" 2024-04-10T22:38:34.895 INFO:teuthology.orchestra.run.smithi044.stdout:ID="centos" 2024-04-10T22:38:34.895 INFO:teuthology.orchestra.run.smithi044.stdout:ID_LIKE="rhel fedora" 2024-04-10T22:38:34.895 INFO:teuthology.orchestra.run.smithi044.stdout:VERSION_ID="8" 2024-04-10T22:38:34.896 INFO:teuthology.orchestra.run.smithi044.stdout:PLATFORM_ID="platform:el8" 2024-04-10T22:38:34.896 INFO:teuthology.orchestra.run.smithi044.stdout:PRETTY_NAME="CentOS Stream 8" 2024-04-10T22:38:34.896 INFO:teuthology.orchestra.run.smithi044.stdout:ANSI_COLOR="0;31" 2024-04-10T22:38:34.896 INFO:teuthology.orchestra.run.smithi044.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2024-04-10T22:38:34.896 INFO:teuthology.orchestra.run.smithi044.stdout:HOME_URL="https://centos.org/" 2024-04-10T22:38:34.896 INFO:teuthology.orchestra.run.smithi044.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-10T22:38:34.896 INFO:teuthology.orchestra.run.smithi044.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2024-04-10T22:38:34.896 INFO:teuthology.orchestra.run.smithi044.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-10T22:38:34.896 INFO:teuthology.lock.ops:Updating smithi044.front.sepia.ceph.com on lock server 2024-04-10T22:38:34.918 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-10T22:38:34.926 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-10T22:38:34.931 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-10T22:38:34.932 DEBUG:teuthology.orchestra.run.smithi027:> test '!' -e /home/ubuntu/cephtest 2024-04-10T22:38:34.934 DEBUG:teuthology.orchestra.run.smithi044:> test '!' -e /home/ubuntu/cephtest 2024-04-10T22:38:34.956 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-10T22:38:34.962 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-10T22:38:34.962 DEBUG:teuthology.orchestra.run.smithi027:> test -z $(ls -A /var/lib/ceph) 2024-04-10T22:38:34.996 DEBUG:teuthology.orchestra.run.smithi044:> test -z $(ls -A /var/lib/ceph) 2024-04-10T22:38:35.023 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-10T22:38:35.069 INFO:teuthology.run_tasks:Running task kernel... 2024-04-10T22:38:35.082 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-04-10T22:38:35.082 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-04-10T22:38:35.083 DEBUG:teuthology.orchestra.run.smithi027:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-10T22:38:35.083 DEBUG:teuthology.orchestra.run.smithi044:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-10T22:38:35.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:38:35.104 DEBUG:teuthology.orchestra.run.smithi044:> uname -r 2024-04-10T22:38:35.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:38:35.105 DEBUG:teuthology.orchestra.run.smithi027:> uname -r 2024-04-10T22:38:35.165 INFO:teuthology.orchestra.run.smithi044.stdout:4.18.0-489.el8.x86_64 2024-04-10T22:38:35.165 INFO:teuthology.task.kernel:Running kernel on smithi044: 4.18.0-489.el8.x86_64 2024-04-10T22:38:35.165 DEBUG:teuthology.orchestra.run.smithi044:> sudo yum install -y kernel 2024-04-10T22:38:35.167 INFO:teuthology.orchestra.run.smithi027.stdout:4.18.0-489.el8.x86_64 2024-04-10T22:38:35.167 INFO:teuthology.task.kernel:Running kernel on smithi027: 4.18.0-489.el8.x86_64 2024-04-10T22:38:35.167 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum install -y kernel 2024-04-10T22:38:39.014 INFO:teuthology.orchestra.run.smithi044.stdout:CentOS Stream 8 - AppStream 23 MB/s | 28 MB 00:01 2024-04-10T22:40:51.030 INFO:teuthology.orchestra.run.smithi027.stdout:CentOS Stream 8 - AppStream 214 kB/s | 28 MB 02:13 2024-04-10T22:40:56.332 INFO:teuthology.orchestra.run.smithi044.stdout:CentOS Stream 8 - BaseOS 80 kB/s | 10 MB 02:12 2024-04-10T22:40:57.930 INFO:teuthology.orchestra.run.smithi027.stdout:CentOS Stream 8 - BaseOS 6.5 MB/s | 10 MB 00:01 2024-04-10T22:40:58.683 INFO:teuthology.orchestra.run.smithi044.stdout:CentOS Stream 8 - Extras 27 kB/s | 18 kB 00:00 2024-04-10T22:41:00.230 INFO:teuthology.orchestra.run.smithi027.stdout:CentOS Stream 8 - Extras 34 kB/s | 18 kB 00:00 2024-04-10T22:41:00.739 INFO:teuthology.orchestra.run.smithi027.stdout:CentOS Stream 8 - Extras common packages 20 kB/s | 7.7 kB 00:00 2024-04-10T22:41:36.327 INFO:teuthology.orchestra.run.smithi027.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:35 2024-04-10T22:41:36.327 INFO:teuthology.orchestra.run.smithi027.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:41:36.328 INFO:teuthology.orchestra.run.smithi027.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:41:36.328 INFO:teuthology.orchestra.run.smithi027.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:41:37.940 INFO:teuthology.orchestra.run.smithi027.stdout:Extra Packages for Enterprise Linux 10 MB/s | 16 MB 00:01 2024-04-10T22:41:41.938 INFO:teuthology.orchestra.run.smithi027.stdout:lab-extras 371 kB/s | 24 kB 00:00 2024-04-10T22:41:42.089 INFO:teuthology.orchestra.run.smithi027.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:41:45.110 INFO:teuthology.orchestra.run.smithi027.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2024-04-10T22:41:45.110 INFO:teuthology.orchestra.run.smithi027.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2024-04-10T22:41:45.223 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2024-04-10T22:41:45.224 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-04-10T22:41:45.224 INFO:teuthology.orchestra.run.smithi027.stdout: Package Architecture Version Repository Size 2024-04-10T22:41:45.224 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-04-10T22:41:45.224 INFO:teuthology.orchestra.run.smithi027.stdout:Installing: 2024-04-10T22:41:45.224 INFO:teuthology.orchestra.run.smithi027.stdout: kernel x86_64 4.18.0-552.el8 baseos 10 M 2024-04-10T22:41:45.224 INFO:teuthology.orchestra.run.smithi027.stdout:Installing dependencies: 2024-04-10T22:41:45.225 INFO:teuthology.orchestra.run.smithi027.stdout: kernel-core x86_64 4.18.0-552.el8 baseos 44 M 2024-04-10T22:41:45.225 INFO:teuthology.orchestra.run.smithi027.stdout: kernel-modules x86_64 4.18.0-552.el8 baseos 36 M 2024-04-10T22:41:45.225 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-04-10T22:41:45.225 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction Summary 2024-04-10T22:41:45.225 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-04-10T22:41:45.225 INFO:teuthology.orchestra.run.smithi027.stdout:Install 3 Packages 2024-04-10T22:41:45.225 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-04-10T22:41:45.226 INFO:teuthology.orchestra.run.smithi027.stdout:Total download size: 90 M 2024-04-10T22:41:45.226 INFO:teuthology.orchestra.run.smithi027.stdout:Installed size: 96 M 2024-04-10T22:41:45.226 INFO:teuthology.orchestra.run.smithi027.stdout:Downloading Packages: 2024-04-10T22:41:48.481 INFO:teuthology.orchestra.run.smithi027.stdout:(1/3): kernel-4.18.0-552.el8.x86_64.rpm 3.4 MB/s | 10 MB 00:03 2024-04-10T22:42:01.871 INFO:teuthology.orchestra.run.smithi027.stdout:(2/3): kernel-modules-4.18.0-552.el8.x86_64.rpm 2.2 MB/s | 36 MB 00:16 2024-04-10T22:42:04.737 INFO:teuthology.orchestra.run.smithi027.stdout:(3/3): kernel-core-4.18.0-552.el8.x86_64.rpm 2.3 MB/s | 44 MB 00:19 2024-04-10T22:42:04.738 INFO:teuthology.orchestra.run.smithi027.stdout:-------------------------------------------------------------------------------- 2024-04-10T22:42:04.738 INFO:teuthology.orchestra.run.smithi027.stdout:Total 4.6 MB/s | 90 MB 00:19 2024-04-10T22:42:05.454 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction check 2024-04-10T22:42:05.908 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction check succeeded. 2024-04-10T22:42:05.908 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction test 2024-04-10T22:42:07.227 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction test succeeded. 2024-04-10T22:42:07.263 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction 2024-04-10T22:42:10.208 INFO:teuthology.orchestra.run.smithi027.stdout: Preparing : 1/1 2024-04-10T22:42:10.776 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:42:14.472 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:42:14.850 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:42:18.533 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:42:18.827 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : kernel-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:42:42.048 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:42:42.073 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:42:42.443 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: kernel-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:42:42.443 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : kernel-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:42:42.443 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : kernel-core-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:42:42.974 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:42:42.975 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-04-10T22:42:42.975 INFO:teuthology.orchestra.run.smithi027.stdout:Installed: 2024-04-10T22:42:42.975 INFO:teuthology.orchestra.run.smithi027.stdout: kernel-4.18.0-552.el8.x86_64 kernel-core-4.18.0-552.el8.x86_64 2024-04-10T22:42:42.975 INFO:teuthology.orchestra.run.smithi027.stdout: kernel-modules-4.18.0-552.el8.x86_64 2024-04-10T22:42:42.975 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-04-10T22:42:42.975 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2024-04-10T22:42:43.149 DEBUG:teuthology.orchestra.run.smithi027:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:42:43.378 INFO:teuthology.orchestra.run.smithi027.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-10T22:42:43.379 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-10T22:42:43.379 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-552.el8.x86_64 2024-04-10T22:42:43.379 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-10T22:42:43.379 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-10T22:42:43.379 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-04-10T22:42:43.379 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi027.front.sepia.ceph.com, path=None, version=distro) 2024-04-10T22:42:43.379 DEBUG:teuthology.orchestra.run.smithi027:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:42:43.600 INFO:teuthology.orchestra.run.smithi027.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-10T22:42:43.600 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-10T22:42:43.600 DEBUG:teuthology.orchestra.run.smithi027:> sudo rpm -qi grub2-tools 2024-04-10T22:42:43.634 INFO:teuthology.orchestra.run.smithi027.stdout:Name : grub2-tools 2024-04-10T22:42:43.634 INFO:teuthology.orchestra.run.smithi027.stdout:Epoch : 1 2024-04-10T22:42:43.634 INFO:teuthology.orchestra.run.smithi027.stdout:Version : 2.02 2024-04-10T22:42:43.635 INFO:teuthology.orchestra.run.smithi027.stdout:Release : 129.el8 2024-04-10T22:42:43.635 INFO:teuthology.orchestra.run.smithi027.stdout:Architecture: x86_64 2024-04-10T22:42:43.635 INFO:teuthology.orchestra.run.smithi027.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2024-04-10T22:42:43.635 INFO:teuthology.orchestra.run.smithi027.stdout:Group : System Environment/Base 2024-04-10T22:42:43.635 INFO:teuthology.orchestra.run.smithi027.stdout:Size : 9265586 2024-04-10T22:42:43.635 INFO:teuthology.orchestra.run.smithi027.stdout:License : GPLv3+ 2024-04-10T22:42:43.635 INFO:teuthology.orchestra.run.smithi027.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2024-04-10T22:42:43.635 INFO:teuthology.orchestra.run.smithi027.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2024-04-10T22:42:43.635 INFO:teuthology.orchestra.run.smithi027.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2024-04-10T22:42:43.635 INFO:teuthology.orchestra.run.smithi027.stdout:Build Host : kbuilder.bsys.centos.org 2024-04-10T22:42:43.635 INFO:teuthology.orchestra.run.smithi027.stdout:Relocations : (not relocatable) 2024-04-10T22:42:43.635 INFO:teuthology.orchestra.run.smithi027.stdout:Packager : CentOS BuildSystem 2024-04-10T22:42:43.635 INFO:teuthology.orchestra.run.smithi027.stdout:Vendor : CentOS 2024-04-10T22:42:43.635 INFO:teuthology.orchestra.run.smithi027.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-10T22:42:43.635 INFO:teuthology.orchestra.run.smithi027.stdout:Summary : Support tools for GRUB. 2024-04-10T22:42:43.636 INFO:teuthology.orchestra.run.smithi027.stdout:Description : 2024-04-10T22:42:43.636 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-04-10T22:42:43.636 INFO:teuthology.orchestra.run.smithi027.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-10T22:42:43.636 INFO:teuthology.orchestra.run.smithi027.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-10T22:42:43.636 INFO:teuthology.orchestra.run.smithi027.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-10T22:42:43.636 INFO:teuthology.orchestra.run.smithi027.stdout:hardware devices. 2024-04-10T22:42:43.636 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-04-10T22:42:43.636 INFO:teuthology.orchestra.run.smithi027.stdout:This subpackage provides tools for support of all platforms. 2024-04-10T22:42:43.637 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-10T22:42:43.637 INFO:teuthology.task.kernel:Updating grub on smithi027 to boot 4.18.0-552.el8.x86_64 2024-04-10T22:42:43.637 DEBUG:teuthology.orchestra.run.smithi027:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-10T22:42:45.247 INFO:teuthology.orchestra.run.smithi027.stderr:Generating grub configuration file ... 2024-04-10T22:42:47.057 INFO:teuthology.orchestra.run.smithi027.stderr:done 2024-04-10T22:42:47.059 DEBUG:teuthology.orchestra.run.smithi027:> mktemp 2024-04-10T22:42:47.089 INFO:teuthology.orchestra.run.smithi027.stdout:/tmp/tmp.nhyqzrfoG0 2024-04-10T22:42:47.089 DEBUG:teuthology.orchestra.run.smithi027:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.nhyqzrfoG0 2024-04-10T22:42:47.163 DEBUG:teuthology.orchestra.run.smithi027:> sudo chmod 0666 /tmp/tmp.nhyqzrfoG0 2024-04-10T22:42:47.311 DEBUG:teuthology.orchestra.remote:smithi027:/tmp/tmp.nhyqzrfoG0 is 6KB 2024-04-10T22:42:47.359 DEBUG:teuthology.orchestra.run.smithi027:> rm -fr /tmp/tmp.nhyqzrfoG0 2024-04-10T22:42:47.378 DEBUG:teuthology.orchestra.run.smithi027:> sudo /bin/ls /boot/loader/entries || true 2024-04-10T22:42:47.461 INFO:teuthology.orchestra.run.smithi027.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2024-04-10T22:42:47.461 INFO:teuthology.orchestra.run.smithi027.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2024-04-10T22:42:47.461 INFO:teuthology.orchestra.run.smithi027.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2024-04-10T22:42:47.461 INFO:teuthology.orchestra.run.smithi027.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64.conf 2024-04-10T22:42:47.462 DEBUG:teuthology.orchestra.run.smithi027:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64 2024-04-10T22:42:47.567 DEBUG:teuthology.orchestra.run.smithi027:> sudo shutdown -r now 2024-04-10T22:43:10.194 INFO:teuthology.orchestra.run.smithi044.stdout:CentOS Stream 8 - Extras common packages 60 B/s | 7.7 kB 02:11 2024-04-10T22:43:17.570 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-04-10T22:43:17.570 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi027.front.sepia.ceph.com' 2024-04-10T22:43:17.571 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi027.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:43:47.987 INFO:teuthology.orchestra.run.smithi044.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:37 2024-04-10T22:43:47.987 INFO:teuthology.orchestra.run.smithi044.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:43:47.987 INFO:teuthology.orchestra.run.smithi044.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:43:47.988 INFO:teuthology.orchestra.run.smithi044.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:43:49.201 INFO:teuthology.orchestra.run.smithi044.stdout:Extra Packages for Enterprise Linux 13 MB/s | 16 MB 00:01 2024-04-10T22:43:52.226 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.27 2024-04-10T22:43:53.186 INFO:teuthology.orchestra.run.smithi044.stdout:lab-extras 320 kB/s | 24 kB 00:00 2024-04-10T22:43:53.342 INFO:teuthology.orchestra.run.smithi044.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:43:56.356 INFO:teuthology.orchestra.run.smithi044.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2024-04-10T22:43:56.356 INFO:teuthology.orchestra.run.smithi044.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2024-04-10T22:43:56.492 INFO:teuthology.orchestra.run.smithi044.stdout:Dependencies resolved. 2024-04-10T22:43:56.493 INFO:teuthology.orchestra.run.smithi044.stdout:================================================================================ 2024-04-10T22:43:56.493 INFO:teuthology.orchestra.run.smithi044.stdout: Package Architecture Version Repository Size 2024-04-10T22:43:56.493 INFO:teuthology.orchestra.run.smithi044.stdout:================================================================================ 2024-04-10T22:43:56.493 INFO:teuthology.orchestra.run.smithi044.stdout:Installing: 2024-04-10T22:43:56.493 INFO:teuthology.orchestra.run.smithi044.stdout: kernel x86_64 4.18.0-552.el8 baseos 10 M 2024-04-10T22:43:56.493 INFO:teuthology.orchestra.run.smithi044.stdout:Installing dependencies: 2024-04-10T22:43:56.493 INFO:teuthology.orchestra.run.smithi044.stdout: kernel-core x86_64 4.18.0-552.el8 baseos 44 M 2024-04-10T22:43:56.493 INFO:teuthology.orchestra.run.smithi044.stdout: kernel-modules x86_64 4.18.0-552.el8 baseos 36 M 2024-04-10T22:43:56.493 INFO:teuthology.orchestra.run.smithi044.stdout: 2024-04-10T22:43:56.494 INFO:teuthology.orchestra.run.smithi044.stdout:Transaction Summary 2024-04-10T22:43:56.494 INFO:teuthology.orchestra.run.smithi044.stdout:================================================================================ 2024-04-10T22:43:56.494 INFO:teuthology.orchestra.run.smithi044.stdout:Install 3 Packages 2024-04-10T22:43:56.494 INFO:teuthology.orchestra.run.smithi044.stdout: 2024-04-10T22:43:56.494 INFO:teuthology.orchestra.run.smithi044.stdout:Total download size: 90 M 2024-04-10T22:43:56.495 INFO:teuthology.orchestra.run.smithi044.stdout:Installed size: 96 M 2024-04-10T22:43:56.495 INFO:teuthology.orchestra.run.smithi044.stdout:Downloading Packages: 2024-04-10T22:44:01.228 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi027.front.sepia.ceph.com' 2024-04-10T22:44:01.229 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi027.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:44:02.676 INFO:teuthology.orchestra.run.smithi044.stdout:(1/3): kernel-4.18.0-552.el8.x86_64.rpm 1.7 MB/s | 10 MB 00:05 2024-04-10T22:44:04.318 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.27 2024-04-10T22:44:12.317 INFO:teuthology.orchestra.run.smithi044.stdout:(2/3): kernel-core-4.18.0-552.el8.x86_64.rpm 2.8 MB/s | 44 MB 00:15 2024-04-10T22:44:15.151 INFO:teuthology.orchestra.run.smithi044.stdout:(3/3): kernel-modules-4.18.0-552.el8.x86_64.rpm 2.0 MB/s | 36 MB 00:18 2024-04-10T22:44:15.151 INFO:teuthology.orchestra.run.smithi044.stdout:-------------------------------------------------------------------------------- 2024-04-10T22:44:15.151 INFO:teuthology.orchestra.run.smithi044.stdout:Total 4.8 MB/s | 90 MB 00:18 2024-04-10T22:44:15.869 INFO:teuthology.orchestra.run.smithi044.stdout:Running transaction check 2024-04-10T22:44:16.310 INFO:teuthology.orchestra.run.smithi044.stdout:Transaction check succeeded. 2024-04-10T22:44:16.310 INFO:teuthology.orchestra.run.smithi044.stdout:Running transaction test 2024-04-10T22:44:16.319 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi027.front.sepia.ceph.com' 2024-04-10T22:44:16.320 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi027.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:44:17.634 INFO:teuthology.orchestra.run.smithi044.stdout:Transaction test succeeded. 2024-04-10T22:44:17.670 INFO:teuthology.orchestra.run.smithi044.stdout:Running transaction 2024-04-10T22:44:19.458 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.27 2024-04-10T22:44:20.627 INFO:teuthology.orchestra.run.smithi044.stdout: Preparing : 1/1 2024-04-10T22:44:21.139 INFO:teuthology.orchestra.run.smithi044.stdout: Installing : kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:44:24.884 INFO:teuthology.orchestra.run.smithi044.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:44:25.247 INFO:teuthology.orchestra.run.smithi044.stdout: Installing : kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:44:28.949 INFO:teuthology.orchestra.run.smithi044.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:44:29.269 INFO:teuthology.orchestra.run.smithi044.stdout: Installing : kernel-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:44:34.459 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi027.front.sepia.ceph.com' 2024-04-10T22:44:34.461 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi027.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:44:52.633 INFO:teuthology.orchestra.run.smithi044.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:44:52.658 INFO:teuthology.orchestra.run.smithi044.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:44:52.894 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.27 2024-04-10T22:44:53.047 INFO:teuthology.orchestra.run.smithi044.stdout: Running scriptlet: kernel-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:44:53.047 INFO:teuthology.orchestra.run.smithi044.stdout: Verifying : kernel-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:44:53.047 INFO:teuthology.orchestra.run.smithi044.stdout: Verifying : kernel-core-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:44:53.557 INFO:teuthology.orchestra.run.smithi044.stdout: Verifying : kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:44:53.558 INFO:teuthology.orchestra.run.smithi044.stdout: 2024-04-10T22:44:53.558 INFO:teuthology.orchestra.run.smithi044.stdout:Installed: 2024-04-10T22:44:53.558 INFO:teuthology.orchestra.run.smithi044.stdout: kernel-4.18.0-552.el8.x86_64 kernel-core-4.18.0-552.el8.x86_64 2024-04-10T22:44:53.558 INFO:teuthology.orchestra.run.smithi044.stdout: kernel-modules-4.18.0-552.el8.x86_64 2024-04-10T22:44:53.558 INFO:teuthology.orchestra.run.smithi044.stdout: 2024-04-10T22:44:53.558 INFO:teuthology.orchestra.run.smithi044.stdout:Complete! 2024-04-10T22:44:53.716 DEBUG:teuthology.orchestra.run.smithi044:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:44:53.947 INFO:teuthology.orchestra.run.smithi044.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-10T22:44:53.947 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-10T22:44:53.948 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-552.el8.x86_64 2024-04-10T22:44:53.948 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-10T22:44:53.948 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-10T22:44:53.948 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-04-10T22:44:53.948 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi044.front.sepia.ceph.com, path=None, version=distro) 2024-04-10T22:44:53.948 DEBUG:teuthology.orchestra.run.smithi044:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:44:54.169 INFO:teuthology.orchestra.run.smithi044.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-10T22:44:54.169 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-10T22:44:54.169 DEBUG:teuthology.orchestra.run.smithi044:> sudo rpm -qi grub2-tools 2024-04-10T22:44:54.203 INFO:teuthology.orchestra.run.smithi044.stdout:Name : grub2-tools 2024-04-10T22:44:54.203 INFO:teuthology.orchestra.run.smithi044.stdout:Epoch : 1 2024-04-10T22:44:54.203 INFO:teuthology.orchestra.run.smithi044.stdout:Version : 2.02 2024-04-10T22:44:54.203 INFO:teuthology.orchestra.run.smithi044.stdout:Release : 129.el8 2024-04-10T22:44:54.204 INFO:teuthology.orchestra.run.smithi044.stdout:Architecture: x86_64 2024-04-10T22:44:54.204 INFO:teuthology.orchestra.run.smithi044.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2024-04-10T22:44:54.204 INFO:teuthology.orchestra.run.smithi044.stdout:Group : System Environment/Base 2024-04-10T22:44:54.204 INFO:teuthology.orchestra.run.smithi044.stdout:Size : 9265586 2024-04-10T22:44:54.204 INFO:teuthology.orchestra.run.smithi044.stdout:License : GPLv3+ 2024-04-10T22:44:54.204 INFO:teuthology.orchestra.run.smithi044.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2024-04-10T22:44:54.204 INFO:teuthology.orchestra.run.smithi044.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2024-04-10T22:44:54.204 INFO:teuthology.orchestra.run.smithi044.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2024-04-10T22:44:54.204 INFO:teuthology.orchestra.run.smithi044.stdout:Build Host : kbuilder.bsys.centos.org 2024-04-10T22:44:54.204 INFO:teuthology.orchestra.run.smithi044.stdout:Relocations : (not relocatable) 2024-04-10T22:44:54.204 INFO:teuthology.orchestra.run.smithi044.stdout:Packager : CentOS BuildSystem 2024-04-10T22:44:54.204 INFO:teuthology.orchestra.run.smithi044.stdout:Vendor : CentOS 2024-04-10T22:44:54.204 INFO:teuthology.orchestra.run.smithi044.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-10T22:44:54.204 INFO:teuthology.orchestra.run.smithi044.stdout:Summary : Support tools for GRUB. 2024-04-10T22:44:54.204 INFO:teuthology.orchestra.run.smithi044.stdout:Description : 2024-04-10T22:44:54.205 INFO:teuthology.orchestra.run.smithi044.stdout: 2024-04-10T22:44:54.205 INFO:teuthology.orchestra.run.smithi044.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-10T22:44:54.205 INFO:teuthology.orchestra.run.smithi044.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-10T22:44:54.205 INFO:teuthology.orchestra.run.smithi044.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-10T22:44:54.205 INFO:teuthology.orchestra.run.smithi044.stdout:hardware devices. 2024-04-10T22:44:54.205 INFO:teuthology.orchestra.run.smithi044.stdout: 2024-04-10T22:44:54.205 INFO:teuthology.orchestra.run.smithi044.stdout:This subpackage provides tools for support of all platforms. 2024-04-10T22:44:54.206 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-10T22:44:54.206 INFO:teuthology.task.kernel:Updating grub on smithi044 to boot 4.18.0-552.el8.x86_64 2024-04-10T22:44:54.206 DEBUG:teuthology.orchestra.run.smithi044:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-10T22:44:55.473 INFO:teuthology.orchestra.run.smithi044.stderr:Generating grub configuration file ... 2024-04-10T22:44:57.460 INFO:teuthology.orchestra.run.smithi044.stderr:done 2024-04-10T22:44:57.461 DEBUG:teuthology.orchestra.run.smithi044:> mktemp 2024-04-10T22:44:57.496 INFO:teuthology.orchestra.run.smithi044.stdout:/tmp/tmp.yY49ICoUZK 2024-04-10T22:44:57.496 DEBUG:teuthology.orchestra.run.smithi044:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.yY49ICoUZK 2024-04-10T22:44:57.568 DEBUG:teuthology.orchestra.run.smithi044:> sudo chmod 0666 /tmp/tmp.yY49ICoUZK 2024-04-10T22:44:57.726 DEBUG:teuthology.orchestra.remote:smithi044:/tmp/tmp.yY49ICoUZK is 6KB 2024-04-10T22:44:57.776 DEBUG:teuthology.orchestra.run.smithi044:> rm -fr /tmp/tmp.yY49ICoUZK 2024-04-10T22:44:57.794 DEBUG:teuthology.orchestra.run.smithi044:> sudo /bin/ls /boot/loader/entries || true 2024-04-10T22:44:57.863 INFO:teuthology.orchestra.run.smithi044.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2024-04-10T22:44:57.863 INFO:teuthology.orchestra.run.smithi044.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2024-04-10T22:44:57.863 INFO:teuthology.orchestra.run.smithi044.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2024-04-10T22:44:57.863 INFO:teuthology.orchestra.run.smithi044.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64.conf 2024-04-10T22:44:57.864 DEBUG:teuthology.orchestra.run.smithi044:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64 2024-04-10T22:44:58.030 DEBUG:teuthology.orchestra.run.smithi044:> sudo shutdown -r now 2024-04-10T22:45:10.907 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi027.front.sepia.ceph.com' 2024-04-10T22:45:10.908 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi027.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:45:12.110 DEBUG:teuthology.orchestra.run.smithi027:> true 2024-04-10T22:45:12.608 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi027.front.sepia.ceph.com' 2024-04-10T22:45:12.608 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "4.18.0-552.el8.x86_64"... 2024-04-10T22:45:12.608 DEBUG:teuthology.orchestra.run.smithi027:> uname -r 2024-04-10T22:45:12.667 INFO:teuthology.orchestra.run.smithi027.stdout:4.18.0-552.el8.x86_64 2024-04-10T22:45:12.667 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-552.el8.x86_64 vs 4.18.0-552.el8.x86_64 2024-04-10T22:45:12.667 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-10T22:45:12.668 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-10T22:45:13.668 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-04-10T22:45:13.669 DEBUG:teuthology.orchestra.run.smithi027:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-10T22:45:13.743 INFO:teuthology.orchestra.run.smithi027.stdout:ttyS1 2024-04-10T22:45:13.757 DEBUG:teuthology.parallel:result is None 2024-04-10T22:45:28.033 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-04-10T22:45:28.034 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi044.front.sepia.ceph.com' 2024-04-10T22:45:28.034 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi044.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:46:28.066 DEBUG:teuthology.orchestra.remote:timed out 2024-04-10T22:46:37.067 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi044.front.sepia.ceph.com' 2024-04-10T22:46:37.069 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi044.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:46:37.604 DEBUG:teuthology.orchestra.run.smithi044:> true 2024-04-10T22:46:37.870 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi044.front.sepia.ceph.com' 2024-04-10T22:46:37.870 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "4.18.0-552.el8.x86_64"... 2024-04-10T22:46:37.870 DEBUG:teuthology.orchestra.run.smithi044:> uname -r 2024-04-10T22:46:37.888 INFO:teuthology.orchestra.run.smithi044.stdout:4.18.0-552.el8.x86_64 2024-04-10T22:46:37.888 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-552.el8.x86_64 vs 4.18.0-552.el8.x86_64 2024-04-10T22:46:37.888 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-10T22:46:37.888 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-10T22:46:38.889 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-04-10T22:46:38.889 DEBUG:teuthology.orchestra.run.smithi044:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-10T22:46:38.961 INFO:teuthology.orchestra.run.smithi044.stdout:ttyS1 2024-04-10T22:46:38.972 DEBUG:teuthology.parallel:result is None 2024-04-10T22:46:38.973 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-10T22:46:38.981 INFO:teuthology.task.internal:Creating test directory... 2024-04-10T22:46:38.981 DEBUG:teuthology.orchestra.run.smithi027:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-10T22:46:38.984 DEBUG:teuthology.orchestra.run.smithi044:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-10T22:46:39.005 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-10T22:46:39.012 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-10T22:46:39.020 INFO:teuthology.task.internal:Creating archive directory... 2024-04-10T22:46:39.020 DEBUG:teuthology.orchestra.run.smithi027:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-10T22:46:39.048 DEBUG:teuthology.orchestra.run.smithi044:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-10T22:46:39.089 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-10T22:46:39.096 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-10T22:46:39.096 DEBUG:teuthology.orchestra.run.smithi027:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-10T22:46:39.127 DEBUG:teuthology.orchestra.run.smithi044:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-10T22:46:39.159 INFO:teuthology.orchestra.run.smithi044.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:46:39.167 INFO:teuthology.orchestra.run.smithi027.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:46:39.170 INFO:teuthology.orchestra.run.smithi044.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:46:39.178 INFO:teuthology.orchestra.run.smithi027.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:46:39.180 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-10T22:46:39.187 INFO:teuthology.task.internal:Configuring sudo... 2024-04-10T22:46:39.187 DEBUG:teuthology.orchestra.run.smithi027:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-10T22:46:39.223 DEBUG:teuthology.orchestra.run.smithi044:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-10T22:46:39.252 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-10T22:46:39.261 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-10T22:46:39.262 DEBUG:teuthology.orchestra.run.smithi027:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-10T22:46:39.295 DEBUG:teuthology.orchestra.run.smithi044:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-10T22:46:39.312 DEBUG:teuthology.orchestra.run.smithi027:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:46:39.410 DEBUG:teuthology.orchestra.run.smithi027:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:46:39.458 DEBUG:teuthology.orchestra.run.smithi027:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:46:39.551 DEBUG:teuthology.orchestra.run.smithi027:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:46:39.587 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-04-10T22:46:39.587 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-10T22:46:39.656 DEBUG:teuthology.orchestra.run.smithi044:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:46:39.709 DEBUG:teuthology.orchestra.run.smithi044:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:46:39.762 DEBUG:teuthology.orchestra.run.smithi044:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:46:39.810 DEBUG:teuthology.orchestra.run.smithi044:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:46:39.849 DEBUG:teuthology.orchestra.run.smithi044:> set -ex 2024-04-10T22:46:39.849 DEBUG:teuthology.orchestra.run.smithi044:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-10T22:46:39.916 DEBUG:teuthology.orchestra.run.smithi027:> sudo service rsyslog restart 2024-04-10T22:46:39.918 DEBUG:teuthology.orchestra.run.smithi044:> sudo service rsyslog restart 2024-04-10T22:46:39.987 INFO:teuthology.orchestra.run.smithi027.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:46:40.067 INFO:teuthology.orchestra.run.smithi044.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:46:40.442 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-10T22:46:40.454 INFO:teuthology.task.internal:Starting timer... 2024-04-10T22:46:40.454 INFO:teuthology.run_tasks:Running task pcp... 2024-04-10T22:46:40.489 INFO:teuthology.run_tasks:Running task selinux... 2024-04-10T22:46:40.498 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-04-10T22:46:40.498 DEBUG:teuthology.orchestra.run.smithi027:> sudo service auditd rotate 2024-04-10T22:46:40.616 INFO:teuthology.orchestra.run.smithi027.stdout:Rotating logs: 2024-04-10T22:46:40.618 DEBUG:teuthology.orchestra.run.smithi044:> sudo service auditd rotate 2024-04-10T22:46:40.915 INFO:teuthology.orchestra.run.smithi044.stdout:Rotating logs: 2024-04-10T22:46:40.928 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-10T22:46:40.929 DEBUG:teuthology.orchestra.run.smithi027:> /usr/sbin/getenforce 2024-04-10T22:46:40.978 INFO:teuthology.orchestra.run.smithi027.stdout:Permissive 2024-04-10T22:46:40.978 DEBUG:teuthology.orchestra.run.smithi044:> /usr/sbin/getenforce 2024-04-10T22:46:41.401 INFO:teuthology.orchestra.run.smithi044.stdout:Permissive 2024-04-10T22:46:41.401 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi027.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi044.front.sepia.ceph.com': 'permissive'} 2024-04-10T22:46:41.401 DEBUG:teuthology.orchestra.run.smithi027:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:46:41.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:46:41.430 DEBUG:teuthology.orchestra.run.smithi044:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:46:42.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:46:42.061 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-10T22:46:42.061 DEBUG:teuthology.orchestra.run.smithi027:> sudo /usr/sbin/setenforce permissive 2024-04-10T22:46:42.100 DEBUG:teuthology.orchestra.run.smithi044:> sudo /usr/sbin/setenforce permissive 2024-04-10T22:46:42.144 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-10T22:46:42.155 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-10T22:46:42.171 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-10T22:46:42.173 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi027.front.sepia.ceph.com,smithi044.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-10T23:01:46.185 INFO:teuthology.task.ansible:Archiving ansible failure log at: /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650174/ansible_failures.yaml 2024-04-10T23:01:46.187 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/__init__.py", line 123, in __enter__ self.begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 508, in begin super(CephLab, self).begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 342, in begin self.execute_playbook() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 374, in execute_playbook self._handle_failure(command, status) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 402, in _handle_failure raise AnsibleFailedError(failures) teuthology.exceptions.AnsibleFailedError: [',', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at'] 2024-04-10T23:01:46.403 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=49d27ad4d12342f486c901f687c9e250 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/__init__.py", line 123, in __enter__ self.begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 508, in begin super(CephLab, self).begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 342, in begin self.execute_playbook() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 374, in execute_playbook self._handle_failure(command, status) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 402, in _handle_failure raise AnsibleFailedError(failures) teuthology.exceptions.AnsibleFailedError: [',', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at'] 2024-04-10T23:01:46.405 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-10T23:01:46.415 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-10T23:01:46.415 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-10T23:01:46.432 DEBUG:teuthology.orchestra.run.smithi027:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-10T23:01:46.516 DEBUG:teuthology.orchestra.run.smithi044:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-10T23:01:46.586 DEBUG:teuthology.orchestra.run.smithi027:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T23:01:46.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T23:01:46.618 DEBUG:teuthology.orchestra.run.smithi044:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T23:01:46.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T23:01:46.651 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-10T23:01:46.659 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-10T23:01:46.666 INFO:teuthology.task.internal:Duration was 906.211926 seconds 2024-04-10T23:01:46.666 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-10T23:01:46.674 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-10T23:01:46.674 DEBUG:teuthology.orchestra.run.smithi027:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-10T23:01:46.676 DEBUG:teuthology.orchestra.run.smithi044:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-10T23:01:46.724 INFO:teuthology.orchestra.run.smithi027.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T23:01:46.742 INFO:teuthology.orchestra.run.smithi044.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T23:01:46.879 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-10T23:01:46.880 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi027.front.sepia.ceph.com 2024-04-10T23:01:46.880 DEBUG:teuthology.orchestra.run.smithi027:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-10T23:01:46.904 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi044.front.sepia.ceph.com 2024-04-10T23:01:46.904 DEBUG:teuthology.orchestra.run.smithi044:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-10T23:01:46.928 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-10T23:01:46.929 DEBUG:teuthology.orchestra.run.smithi027:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-10T23:01:46.946 DEBUG:teuthology.orchestra.run.smithi044:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-10T23:01:47.012 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-10T23:01:47.012 DEBUG:teuthology.orchestra.run.smithi027:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-10T23:01:47.028 DEBUG:teuthology.orchestra.run.smithi044:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-10T23:01:47.125 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-10T23:01:47.134 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-10T23:01:47.134 DEBUG:teuthology.orchestra.run.smithi027:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-10T23:01:47.161 DEBUG:teuthology.orchestra.run.smithi044:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-10T23:01:47.189 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-10T23:01:47.197 DEBUG:teuthology.orchestra.run.smithi027:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-10T23:01:47.204 DEBUG:teuthology.orchestra.run.smithi044:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-10T23:01:47.229 INFO:teuthology.orchestra.run.smithi027.stdout:kernel.core_pattern = core 2024-04-10T23:01:47.256 INFO:teuthology.orchestra.run.smithi044.stdout:kernel.core_pattern = core 2024-04-10T23:01:47.279 DEBUG:teuthology.orchestra.run.smithi027:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-10T23:01:47.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T23:01:47.307 DEBUG:teuthology.orchestra.run.smithi044:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-10T23:01:47.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T23:01:47.336 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-10T23:01:47.344 INFO:teuthology.task.internal:Transferring archived files... 2024-04-10T23:01:47.346 DEBUG:teuthology.misc:Transferring archived files from smithi027:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650174/remote/smithi027 2024-04-10T23:01:47.347 DEBUG:teuthology.orchestra.run.smithi027:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-10T23:01:47.403 DEBUG:teuthology.misc:Transferring archived files from smithi044:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650174/remote/smithi044 2024-04-10T23:01:47.404 DEBUG:teuthology.orchestra.run.smithi044:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-10T23:01:47.495 INFO:teuthology.task.internal:Removing archive directory... 2024-04-10T23:01:47.495 DEBUG:teuthology.orchestra.run.smithi027:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-10T23:01:47.497 DEBUG:teuthology.orchestra.run.smithi044:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-10T23:01:47.515 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-10T23:01:47.525 INFO:teuthology.task.internal:Not uploading archives. 2024-04-10T23:01:47.526 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-10T23:01:47.547 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-10T23:01:47.547 DEBUG:teuthology.orchestra.run.smithi027:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-10T23:01:47.555 DEBUG:teuthology.orchestra.run.smithi044:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-10T23:01:47.571 INFO:teuthology.orchestra.run.smithi027.stdout: 393224 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 10 23:01 /home/ubuntu/cephtest 2024-04-10T23:01:47.573 INFO:teuthology.orchestra.run.smithi044.stdout: 525941 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 10 23:01 /home/ubuntu/cephtest 2024-04-10T23:01:47.574 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-10T23:01:47.583 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-10T23:01:47.600 INFO:teuthology.nuke:Checking targets against current locks 2024-04-10T23:01:47.632 DEBUG:teuthology.nuke:shortname: smithi027 2024-04-10T23:01:47.632 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T23:01:47.682 DEBUG:teuthology.nuke:shortname: smithi044 2024-04-10T23:01:47.682 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T23:01:47.697 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi027.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650174', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:13.075234', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOznJROncvZ/sl4kvt1uMFmXbcvtyJA1c/fZSZQ8Sg2B4kUlYYsaE/1Z4ivcUVXCNmusjHqAExDZ5P5U2csuf4k='} 2024-04-10T23:01:47.745 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi044.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650174', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:13.073913', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPxi/gvs1PpXnaMk8aKlR0U7WUu8169ktzHlQf7DjusykuABHNXGrpmKRK+S2aTz6JCmi8Mu1OE/mIAfT3G5MlM='} 2024-04-10T23:01:47.784 INFO:teuthology.orchestra.console.smithi027:Power off 2024-04-10T23:01:47.784 DEBUG:teuthology.orchestra.console.smithi027:pexpect command: ipmitool -H smithi027.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-10T23:01:47.838 INFO:teuthology.orchestra.console.smithi044:Power off 2024-04-10T23:01:47.838 DEBUG:teuthology.orchestra.console.smithi044:pexpect command: ipmitool -H smithi044.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-10T23:01:47.846 DEBUG:teuthology.orchestra.console.smithi027:power off output: Chassis Power Control: Down/Off 2024-04-10T23:01:47.846 DEBUG:teuthology.orchestra.console.smithi027:pexpect command: ipmitool -H smithi027.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:01:47.889 DEBUG:teuthology.orchestra.console.smithi044:power off output: Chassis Power Control: Down/Off 2024-04-10T23:01:47.889 DEBUG:teuthology.orchestra.console.smithi044:pexpect command: ipmitool -H smithi044.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:01:47.905 DEBUG:teuthology.orchestra.console.smithi027:check power output: Chassis Power is on 2024-04-10T23:01:47.951 DEBUG:teuthology.orchestra.console.smithi044:check power output: Chassis Power is on 2024-04-10T23:01:51.906 DEBUG:teuthology.orchestra.console.smithi027:pexpect command: ipmitool -H smithi027.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:01:51.952 DEBUG:teuthology.orchestra.console.smithi044:pexpect command: ipmitool -H smithi044.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:01:52.016 DEBUG:teuthology.orchestra.console.smithi027:check power output: Chassis Power is on 2024-04-10T23:01:52.061 DEBUG:teuthology.orchestra.console.smithi044:check power output: Chassis Power is on 2024-04-10T23:01:56.017 DEBUG:teuthology.orchestra.console.smithi027:pexpect command: ipmitool -H smithi027.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:01:56.062 DEBUG:teuthology.orchestra.console.smithi044:pexpect command: ipmitool -H smithi044.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:01:56.128 DEBUG:teuthology.orchestra.console.smithi027:check power output: Chassis Power is on 2024-04-10T23:01:56.172 DEBUG:teuthology.orchestra.console.smithi044:check power output: Chassis Power is off 2024-04-10T23:01:56.272 INFO:teuthology.orchestra.console.smithi044:Power off completed 2024-04-10T23:02:00.133 DEBUG:teuthology.orchestra.console.smithi027:pexpect command: ipmitool -H smithi027.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:02:00.242 DEBUG:teuthology.orchestra.console.smithi027:check power output: Chassis Power is off 2024-04-10T23:02:00.343 INFO:teuthology.orchestra.console.smithi027:Power off completed 2024-04-10T23:02:00.448 INFO:teuthology.run:Summary data: description: upgrade:pacific-x/stress-split/{0-distro/centos_8.stream_container_tools_crun 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} duration: 906.2119262218475 failure_reason: '['','', ''Bugzilla. In case of problems, contact the owner of this repository.'', ''Enabling a Copr repository. Please note that this repository is not part'', ''Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable'', ''Please do not file bug reports about these packages in Fedora'', ''The Fedora Project does not exercise any power over the contents of'', ''and packages are not held to any quality or security level.'', ''of the main distribution, and quality may vary.'', ''this repository beyond the rules outlined in the Copr FAQ at'']' owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=49d27ad4d12342f486c901f687c9e250 status: dead success: false 2024-04-10T23:02:00.449 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-10T23:02:00.551 INFO:teuthology.run:DEAD