2024-04-10T22:38:55.829 INFO:root:teuthology version: 0.0.1.dev267+g6c63784 2024-04-10T22:38:55.829 DEBUG:teuthology.run:Teuthology command: teuthology --name teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi --owner scheduled_teuthology@teuthology --archive /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650198 --verbose --description upgrade:pacific-x/stress-split/{0-distro/centos_8.stream_container_tools 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} -- /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650198/orig.config.yaml 2024-04-10T22:38:55.885 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-10T22:38:55.971 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650198 branch: reef description: upgrade:pacific-x/stress-split/{0-distro/centos_8.stream_container_tools 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} email: ceph-qa@ceph.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' first_in_suite: false job_id: '7650198' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 8.stream overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: reef sha1: 4f70a9a90881107996c8b7a3e89ba8b32624713c owner: scheduled_teuthology@teuthology priority: 830 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 4604 sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: pacific clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 0/1 suite: upgrade:pacific-x suite_branch: reef suite_path: /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 4f70a9a90881107996c8b7a3e89ba8b32624713c targets: smithi031.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNKFMx3gdxQeHDIKftIK2KVWZQ5YFl0fQtYGhwCVyPNgZfopwhxtBUWQ3yTgCqnUKvSRAn0Tuw5VsUaKo4lTCm4= smithi053.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFScDMQ5OzYa1CXENdMQJKk1XXUxkoFu6bwIrp3FWsiGFOFGwB8v4DpPrFT7LHPgPfgNtZUZuIQlbT3CmaGRyE0= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - install: branch: pacific exclude_packages: - ceph-volume - cephadm: cephadm_branch: pacific cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:pacific - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 6c637841c215537a4502385240412f1966e0faab timestamp: 2024-04-10_01:08:08 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-04-10T22:38:55.971 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa; will attempt to use it 2024-04-10T22:38:55.972 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa/tasks 2024-04-10T22:38:55.972 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-10T22:38:55.974 INFO:teuthology.task.internal:Checking packages... 2024-04-10T22:38:55.994 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'd540ebaca6b131a1dd560e7f69e024b133bbaa42' 2024-04-10T22:38:55.994 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-10T22:38:55.994 INFO:teuthology.packaging:ref: None 2024-04-10T22:38:55.994 INFO:teuthology.packaging:tag: None 2024-04-10T22:38:55.994 INFO:teuthology.packaging:branch: reef 2024-04-10T22:38:55.994 INFO:teuthology.packaging:sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 2024-04-10T22:38:55.995 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef 2024-04-10T22:38:56.359 INFO:teuthology.task.internal:Found packages for ceph version 18.2.2-1189.gc96c8d47 2024-04-10T22:38:56.360 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-10T22:38:56.366 INFO:teuthology.task.internal:no buildpackages task found 2024-04-10T22:38:56.367 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-10T22:38:56.374 INFO:teuthology.task.internal:Saving configuration 2024-04-10T22:38:56.394 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-10T22:38:56.400 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T22:38:56.423 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi031.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650198', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:34:33.358071', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNKFMx3gdxQeHDIKftIK2KVWZQ5YFl0fQtYGhwCVyPNgZfopwhxtBUWQ3yTgCqnUKvSRAn0Tuw5VsUaKo4lTCm4='} 2024-04-10T22:38:56.441 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi053.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650198', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:34:33.356032', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFScDMQ5OzYa1CXENdMQJKk1XXUxkoFu6bwIrp3FWsiGFOFGwB8v4DpPrFT7LHPgPfgNtZUZuIQlbT3CmaGRyE0='} 2024-04-10T22:38:56.441 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-10T22:38:56.448 INFO:teuthology.task.internal:roles: ubuntu@smithi031.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-04-10T22:38:56.448 INFO:teuthology.task.internal:roles: ubuntu@smithi053.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-04-10T22:38:56.448 INFO:teuthology.run_tasks:Running task console_log... 2024-04-10T22:38:56.520 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f4799d860d0>, signals=[15]) 2024-04-10T22:38:56.520 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-10T22:38:56.526 INFO:teuthology.task.internal:Opening connections... 2024-04-10T22:38:56.526 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi031.front.sepia.ceph.com 2024-04-10T22:38:56.528 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi031.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:38:56.603 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi053.front.sepia.ceph.com 2024-04-10T22:38:56.604 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi053.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:38:56.677 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-10T22:38:56.684 DEBUG:teuthology.orchestra.run.smithi031:> uname -m 2024-04-10T22:38:56.707 INFO:teuthology.orchestra.run.smithi031.stdout:x86_64 2024-04-10T22:38:56.707 DEBUG:teuthology.orchestra.run.smithi031:> cat /etc/os-release 2024-04-10T22:38:56.769 INFO:teuthology.orchestra.run.smithi031.stdout:NAME="CentOS Stream" 2024-04-10T22:38:56.770 INFO:teuthology.orchestra.run.smithi031.stdout:VERSION="8" 2024-04-10T22:38:56.770 INFO:teuthology.orchestra.run.smithi031.stdout:ID="centos" 2024-04-10T22:38:56.770 INFO:teuthology.orchestra.run.smithi031.stdout:ID_LIKE="rhel fedora" 2024-04-10T22:38:56.770 INFO:teuthology.orchestra.run.smithi031.stdout:VERSION_ID="8" 2024-04-10T22:38:56.770 INFO:teuthology.orchestra.run.smithi031.stdout:PLATFORM_ID="platform:el8" 2024-04-10T22:38:56.770 INFO:teuthology.orchestra.run.smithi031.stdout:PRETTY_NAME="CentOS Stream 8" 2024-04-10T22:38:56.770 INFO:teuthology.orchestra.run.smithi031.stdout:ANSI_COLOR="0;31" 2024-04-10T22:38:56.770 INFO:teuthology.orchestra.run.smithi031.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2024-04-10T22:38:56.770 INFO:teuthology.orchestra.run.smithi031.stdout:HOME_URL="https://centos.org/" 2024-04-10T22:38:56.770 INFO:teuthology.orchestra.run.smithi031.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-10T22:38:56.770 INFO:teuthology.orchestra.run.smithi031.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2024-04-10T22:38:56.770 INFO:teuthology.orchestra.run.smithi031.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-10T22:38:56.771 INFO:teuthology.lock.ops:Updating smithi031.front.sepia.ceph.com on lock server 2024-04-10T22:38:56.791 DEBUG:teuthology.orchestra.run.smithi053:> uname -m 2024-04-10T22:38:56.814 INFO:teuthology.orchestra.run.smithi053.stdout:x86_64 2024-04-10T22:38:56.814 DEBUG:teuthology.orchestra.run.smithi053:> cat /etc/os-release 2024-04-10T22:38:56.875 INFO:teuthology.orchestra.run.smithi053.stdout:NAME="CentOS Stream" 2024-04-10T22:38:56.875 INFO:teuthology.orchestra.run.smithi053.stdout:VERSION="8" 2024-04-10T22:38:56.875 INFO:teuthology.orchestra.run.smithi053.stdout:ID="centos" 2024-04-10T22:38:56.875 INFO:teuthology.orchestra.run.smithi053.stdout:ID_LIKE="rhel fedora" 2024-04-10T22:38:56.876 INFO:teuthology.orchestra.run.smithi053.stdout:VERSION_ID="8" 2024-04-10T22:38:56.876 INFO:teuthology.orchestra.run.smithi053.stdout:PLATFORM_ID="platform:el8" 2024-04-10T22:38:56.876 INFO:teuthology.orchestra.run.smithi053.stdout:PRETTY_NAME="CentOS Stream 8" 2024-04-10T22:38:56.876 INFO:teuthology.orchestra.run.smithi053.stdout:ANSI_COLOR="0;31" 2024-04-10T22:38:56.876 INFO:teuthology.orchestra.run.smithi053.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2024-04-10T22:38:56.876 INFO:teuthology.orchestra.run.smithi053.stdout:HOME_URL="https://centos.org/" 2024-04-10T22:38:56.876 INFO:teuthology.orchestra.run.smithi053.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-10T22:38:56.876 INFO:teuthology.orchestra.run.smithi053.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2024-04-10T22:38:56.876 INFO:teuthology.orchestra.run.smithi053.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-10T22:38:56.876 INFO:teuthology.lock.ops:Updating smithi053.front.sepia.ceph.com on lock server 2024-04-10T22:38:56.907 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-10T22:38:56.917 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-10T22:38:56.922 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-10T22:38:56.923 DEBUG:teuthology.orchestra.run.smithi031:> test '!' -e /home/ubuntu/cephtest 2024-04-10T22:38:56.925 DEBUG:teuthology.orchestra.run.smithi053:> test '!' -e /home/ubuntu/cephtest 2024-04-10T22:38:56.946 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-10T22:38:56.952 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-10T22:38:56.952 DEBUG:teuthology.orchestra.run.smithi031:> test -z $(ls -A /var/lib/ceph) 2024-04-10T22:38:56.987 DEBUG:teuthology.orchestra.run.smithi053:> test -z $(ls -A /var/lib/ceph) 2024-04-10T22:38:57.013 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-10T22:38:57.066 INFO:teuthology.run_tasks:Running task kernel... 2024-04-10T22:38:57.081 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-04-10T22:38:57.081 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-04-10T22:38:57.081 DEBUG:teuthology.orchestra.run.smithi031:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-10T22:38:57.082 DEBUG:teuthology.orchestra.run.smithi053:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-10T22:38:57.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:38:57.103 DEBUG:teuthology.orchestra.run.smithi053:> uname -r 2024-04-10T22:38:57.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:38:57.104 DEBUG:teuthology.orchestra.run.smithi031:> uname -r 2024-04-10T22:38:57.165 INFO:teuthology.orchestra.run.smithi053.stdout:4.18.0-489.el8.x86_64 2024-04-10T22:38:57.165 INFO:teuthology.task.kernel:Running kernel on smithi053: 4.18.0-489.el8.x86_64 2024-04-10T22:38:57.166 DEBUG:teuthology.orchestra.run.smithi053:> sudo yum install -y kernel 2024-04-10T22:38:57.167 INFO:teuthology.orchestra.run.smithi031.stdout:4.18.0-489.el8.x86_64 2024-04-10T22:38:57.167 INFO:teuthology.task.kernel:Running kernel on smithi031: 4.18.0-489.el8.x86_64 2024-04-10T22:38:57.167 DEBUG:teuthology.orchestra.run.smithi031:> sudo yum install -y kernel 2024-04-10T22:39:01.508 INFO:teuthology.orchestra.run.smithi053.stdout:CentOS Stream 8 - AppStream 16 MB/s | 28 MB 00:01 2024-04-10T22:39:01.955 INFO:teuthology.orchestra.run.smithi031.stdout:CentOS Stream 8 - AppStream 12 MB/s | 28 MB 00:02 2024-04-10T22:39:07.875 INFO:teuthology.orchestra.run.smithi053.stdout:CentOS Stream 8 - BaseOS 9.6 MB/s | 10 MB 00:01 2024-04-10T22:39:08.230 INFO:teuthology.orchestra.run.smithi031.stdout:CentOS Stream 8 - BaseOS 10 MB/s | 10 MB 00:01 2024-04-10T22:41:21.254 INFO:teuthology.orchestra.run.smithi031.stdout:CentOS Stream 8 - Extras 139 B/s | 18 kB 02:11 2024-04-10T22:41:21.754 INFO:teuthology.orchestra.run.smithi031.stdout:CentOS Stream 8 - Extras common packages 20 kB/s | 7.7 kB 00:00 2024-04-10T22:41:21.845 INFO:teuthology.orchestra.run.smithi053.stdout:CentOS Stream 8 - Extras 138 B/s | 18 kB 02:12 2024-04-10T22:41:22.304 INFO:teuthology.orchestra.run.smithi053.stdout:CentOS Stream 8 - Extras common packages 23 kB/s | 7.7 kB 00:00 2024-04-10T22:42:04.963 INFO:teuthology.orchestra.run.smithi031.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:43 2024-04-10T22:42:04.964 INFO:teuthology.orchestra.run.smithi031.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:42:04.964 INFO:teuthology.orchestra.run.smithi031.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:42:04.964 INFO:teuthology.orchestra.run.smithi031.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:42:06.032 INFO:teuthology.orchestra.run.smithi031.stdout:Extra Packages for Enterprise Linux 15 MB/s | 16 MB 00:01 2024-04-10T22:42:06.903 INFO:teuthology.orchestra.run.smithi053.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:44 2024-04-10T22:42:06.904 INFO:teuthology.orchestra.run.smithi053.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:42:06.904 INFO:teuthology.orchestra.run.smithi053.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:42:06.904 INFO:teuthology.orchestra.run.smithi053.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:42:09.909 INFO:teuthology.orchestra.run.smithi053.stdout:Extra Packages for Enterprise Linux 5.4 MB/s | 16 MB 00:03 2024-04-10T22:42:10.049 INFO:teuthology.orchestra.run.smithi031.stdout:lab-extras 438 kB/s | 24 kB 00:00 2024-04-10T22:42:10.200 INFO:teuthology.orchestra.run.smithi031.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:42:13.137 INFO:teuthology.orchestra.run.smithi031.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2024-04-10T22:42:13.138 INFO:teuthology.orchestra.run.smithi031.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2024-04-10T22:42:13.245 INFO:teuthology.orchestra.run.smithi031.stdout:Dependencies resolved. 2024-04-10T22:42:13.246 INFO:teuthology.orchestra.run.smithi031.stdout:================================================================================ 2024-04-10T22:42:13.246 INFO:teuthology.orchestra.run.smithi031.stdout: Package Architecture Version Repository Size 2024-04-10T22:42:13.246 INFO:teuthology.orchestra.run.smithi031.stdout:================================================================================ 2024-04-10T22:42:13.246 INFO:teuthology.orchestra.run.smithi031.stdout:Installing: 2024-04-10T22:42:13.247 INFO:teuthology.orchestra.run.smithi031.stdout: kernel x86_64 4.18.0-552.el8 baseos 10 M 2024-04-10T22:42:13.247 INFO:teuthology.orchestra.run.smithi031.stdout:Installing dependencies: 2024-04-10T22:42:13.247 INFO:teuthology.orchestra.run.smithi031.stdout: kernel-core x86_64 4.18.0-552.el8 baseos 44 M 2024-04-10T22:42:13.247 INFO:teuthology.orchestra.run.smithi031.stdout: kernel-modules x86_64 4.18.0-552.el8 baseos 36 M 2024-04-10T22:42:13.247 INFO:teuthology.orchestra.run.smithi031.stdout: 2024-04-10T22:42:13.247 INFO:teuthology.orchestra.run.smithi031.stdout:Transaction Summary 2024-04-10T22:42:13.247 INFO:teuthology.orchestra.run.smithi031.stdout:================================================================================ 2024-04-10T22:42:13.247 INFO:teuthology.orchestra.run.smithi031.stdout:Install 3 Packages 2024-04-10T22:42:13.247 INFO:teuthology.orchestra.run.smithi031.stdout: 2024-04-10T22:42:13.248 INFO:teuthology.orchestra.run.smithi031.stdout:Total download size: 90 M 2024-04-10T22:42:13.248 INFO:teuthology.orchestra.run.smithi031.stdout:Installed size: 96 M 2024-04-10T22:42:13.248 INFO:teuthology.orchestra.run.smithi031.stdout:Downloading Packages: 2024-04-10T22:42:13.925 INFO:teuthology.orchestra.run.smithi053.stdout:lab-extras 359 kB/s | 24 kB 00:00 2024-04-10T22:42:14.077 INFO:teuthology.orchestra.run.smithi053.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:42:17.047 INFO:teuthology.orchestra.run.smithi053.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2024-04-10T22:42:17.048 INFO:teuthology.orchestra.run.smithi053.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2024-04-10T22:42:17.150 INFO:teuthology.orchestra.run.smithi053.stdout:Dependencies resolved. 2024-04-10T22:42:17.151 INFO:teuthology.orchestra.run.smithi053.stdout:================================================================================ 2024-04-10T22:42:17.151 INFO:teuthology.orchestra.run.smithi053.stdout: Package Architecture Version Repository Size 2024-04-10T22:42:17.151 INFO:teuthology.orchestra.run.smithi053.stdout:================================================================================ 2024-04-10T22:42:17.152 INFO:teuthology.orchestra.run.smithi053.stdout:Installing: 2024-04-10T22:42:17.152 INFO:teuthology.orchestra.run.smithi053.stdout: kernel x86_64 4.18.0-552.el8 baseos 10 M 2024-04-10T22:42:17.152 INFO:teuthology.orchestra.run.smithi053.stdout:Installing dependencies: 2024-04-10T22:42:17.152 INFO:teuthology.orchestra.run.smithi053.stdout: kernel-core x86_64 4.18.0-552.el8 baseos 44 M 2024-04-10T22:42:17.152 INFO:teuthology.orchestra.run.smithi053.stdout: kernel-modules x86_64 4.18.0-552.el8 baseos 36 M 2024-04-10T22:42:17.152 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-04-10T22:42:17.152 INFO:teuthology.orchestra.run.smithi053.stdout:Transaction Summary 2024-04-10T22:42:17.152 INFO:teuthology.orchestra.run.smithi053.stdout:================================================================================ 2024-04-10T22:42:17.152 INFO:teuthology.orchestra.run.smithi053.stdout:Install 3 Packages 2024-04-10T22:42:17.152 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-04-10T22:42:17.153 INFO:teuthology.orchestra.run.smithi053.stdout:Total download size: 90 M 2024-04-10T22:42:17.153 INFO:teuthology.orchestra.run.smithi053.stdout:Installed size: 96 M 2024-04-10T22:42:17.153 INFO:teuthology.orchestra.run.smithi053.stdout:Downloading Packages: 2024-04-10T22:44:26.172 INFO:teuthology.orchestra.run.smithi031.stdout:(1/3): kernel-4.18.0-552.el8.x86_64.rpm 12 MB/s | 10 MB 00:00 2024-04-10T22:44:34.861 INFO:teuthology.orchestra.run.smithi053.stdout:(1/3): kernel-4.18.0-552.el8.x86_64.rpm 1.9 MB/s | 10 MB 00:05 2024-04-10T22:44:36.661 INFO:teuthology.orchestra.run.smithi053.stdout:(2/3): kernel-modules-4.18.0-552.el8.x86_64.rpm 5.1 MB/s | 36 MB 00:07 2024-04-10T22:44:38.612 INFO:teuthology.orchestra.run.smithi031.stdout:(2/3): kernel-modules-4.18.0-552.el8.x86_64.rpm 2.7 MB/s | 36 MB 00:13 2024-04-10T22:44:39.300 INFO:teuthology.orchestra.run.smithi053.stdout:(3/3): kernel-core-4.18.0-552.el8.x86_64.rpm 4.5 MB/s | 44 MB 00:09 2024-04-10T22:44:39.300 INFO:teuthology.orchestra.run.smithi053.stdout:-------------------------------------------------------------------------------- 2024-04-10T22:44:39.300 INFO:teuthology.orchestra.run.smithi053.stdout:Total 652 kB/s | 90 MB 02:22 2024-04-10T22:44:40.018 INFO:teuthology.orchestra.run.smithi053.stdout:Running transaction check 2024-04-10T22:44:40.478 INFO:teuthology.orchestra.run.smithi053.stdout:Transaction check succeeded. 2024-04-10T22:44:40.478 INFO:teuthology.orchestra.run.smithi053.stdout:Running transaction test 2024-04-10T22:44:41.810 INFO:teuthology.orchestra.run.smithi053.stdout:Transaction test succeeded. 2024-04-10T22:44:41.838 INFO:teuthology.orchestra.run.smithi053.stdout:Running transaction 2024-04-10T22:44:44.802 INFO:teuthology.orchestra.run.smithi053.stdout: Preparing : 1/1 2024-04-10T22:44:45.385 INFO:teuthology.orchestra.run.smithi053.stdout: Installing : kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:44:49.155 INFO:teuthology.orchestra.run.smithi053.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:44:49.494 INFO:teuthology.orchestra.run.smithi031.stdout:(3/3): kernel-core-4.18.0-552.el8.x86_64.rpm 1.8 MB/s | 44 MB 00:24 2024-04-10T22:44:49.494 INFO:teuthology.orchestra.run.smithi031.stdout:-------------------------------------------------------------------------------- 2024-04-10T22:44:49.494 INFO:teuthology.orchestra.run.smithi031.stdout:Total 593 kB/s | 90 MB 02:36 2024-04-10T22:44:49.543 INFO:teuthology.orchestra.run.smithi053.stdout: Installing : kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:44:50.212 INFO:teuthology.orchestra.run.smithi031.stdout:Running transaction check 2024-04-10T22:44:50.654 INFO:teuthology.orchestra.run.smithi031.stdout:Transaction check succeeded. 2024-04-10T22:44:50.654 INFO:teuthology.orchestra.run.smithi031.stdout:Running transaction test 2024-04-10T22:44:51.964 INFO:teuthology.orchestra.run.smithi031.stdout:Transaction test succeeded. 2024-04-10T22:44:51.991 INFO:teuthology.orchestra.run.smithi031.stdout:Running transaction 2024-04-10T22:44:53.226 INFO:teuthology.orchestra.run.smithi053.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:44:53.521 INFO:teuthology.orchestra.run.smithi053.stdout: Installing : kernel-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:44:54.941 INFO:teuthology.orchestra.run.smithi031.stdout: Preparing : 1/1 2024-04-10T22:44:55.445 INFO:teuthology.orchestra.run.smithi031.stdout: Installing : kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:44:59.140 INFO:teuthology.orchestra.run.smithi031.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:44:59.469 INFO:teuthology.orchestra.run.smithi031.stdout: Installing : kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:45:03.155 INFO:teuthology.orchestra.run.smithi031.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:45:03.438 INFO:teuthology.orchestra.run.smithi031.stdout: Installing : kernel-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:45:16.931 INFO:teuthology.orchestra.run.smithi053.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:45:16.954 INFO:teuthology.orchestra.run.smithi053.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:45:17.323 INFO:teuthology.orchestra.run.smithi053.stdout: Running scriptlet: kernel-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:45:17.323 INFO:teuthology.orchestra.run.smithi053.stdout: Verifying : kernel-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:45:17.323 INFO:teuthology.orchestra.run.smithi053.stdout: Verifying : kernel-core-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:45:17.872 INFO:teuthology.orchestra.run.smithi053.stdout: Verifying : kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:45:17.872 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-04-10T22:45:17.872 INFO:teuthology.orchestra.run.smithi053.stdout:Installed: 2024-04-10T22:45:17.873 INFO:teuthology.orchestra.run.smithi053.stdout: kernel-4.18.0-552.el8.x86_64 kernel-core-4.18.0-552.el8.x86_64 2024-04-10T22:45:17.873 INFO:teuthology.orchestra.run.smithi053.stdout: kernel-modules-4.18.0-552.el8.x86_64 2024-04-10T22:45:17.873 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-04-10T22:45:17.873 INFO:teuthology.orchestra.run.smithi053.stdout:Complete! 2024-04-10T22:45:18.029 DEBUG:teuthology.orchestra.run.smithi053:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:45:18.259 INFO:teuthology.orchestra.run.smithi053.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-10T22:45:18.259 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-10T22:45:18.259 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-552.el8.x86_64 2024-04-10T22:45:18.259 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-10T22:45:18.259 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-10T22:45:18.260 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-04-10T22:45:18.260 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi053.front.sepia.ceph.com, path=None, version=distro) 2024-04-10T22:45:18.260 DEBUG:teuthology.orchestra.run.smithi053:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:45:18.481 INFO:teuthology.orchestra.run.smithi053.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-10T22:45:18.481 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-10T22:45:18.482 DEBUG:teuthology.orchestra.run.smithi053:> sudo rpm -qi grub2-tools 2024-04-10T22:45:18.516 INFO:teuthology.orchestra.run.smithi053.stdout:Name : grub2-tools 2024-04-10T22:45:18.516 INFO:teuthology.orchestra.run.smithi053.stdout:Epoch : 1 2024-04-10T22:45:18.516 INFO:teuthology.orchestra.run.smithi053.stdout:Version : 2.02 2024-04-10T22:45:18.516 INFO:teuthology.orchestra.run.smithi053.stdout:Release : 129.el8 2024-04-10T22:45:18.517 INFO:teuthology.orchestra.run.smithi053.stdout:Architecture: x86_64 2024-04-10T22:45:18.517 INFO:teuthology.orchestra.run.smithi053.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2024-04-10T22:45:18.517 INFO:teuthology.orchestra.run.smithi053.stdout:Group : System Environment/Base 2024-04-10T22:45:18.517 INFO:teuthology.orchestra.run.smithi053.stdout:Size : 9265586 2024-04-10T22:45:18.517 INFO:teuthology.orchestra.run.smithi053.stdout:License : GPLv3+ 2024-04-10T22:45:18.517 INFO:teuthology.orchestra.run.smithi053.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2024-04-10T22:45:18.517 INFO:teuthology.orchestra.run.smithi053.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2024-04-10T22:45:18.517 INFO:teuthology.orchestra.run.smithi053.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2024-04-10T22:45:18.517 INFO:teuthology.orchestra.run.smithi053.stdout:Build Host : kbuilder.bsys.centos.org 2024-04-10T22:45:18.517 INFO:teuthology.orchestra.run.smithi053.stdout:Relocations : (not relocatable) 2024-04-10T22:45:18.517 INFO:teuthology.orchestra.run.smithi053.stdout:Packager : CentOS BuildSystem 2024-04-10T22:45:18.517 INFO:teuthology.orchestra.run.smithi053.stdout:Vendor : CentOS 2024-04-10T22:45:18.517 INFO:teuthology.orchestra.run.smithi053.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-10T22:45:18.518 INFO:teuthology.orchestra.run.smithi053.stdout:Summary : Support tools for GRUB. 2024-04-10T22:45:18.518 INFO:teuthology.orchestra.run.smithi053.stdout:Description : 2024-04-10T22:45:18.518 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-04-10T22:45:18.518 INFO:teuthology.orchestra.run.smithi053.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-10T22:45:18.518 INFO:teuthology.orchestra.run.smithi053.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-10T22:45:18.518 INFO:teuthology.orchestra.run.smithi053.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-10T22:45:18.518 INFO:teuthology.orchestra.run.smithi053.stdout:hardware devices. 2024-04-10T22:45:18.518 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-04-10T22:45:18.518 INFO:teuthology.orchestra.run.smithi053.stdout:This subpackage provides tools for support of all platforms. 2024-04-10T22:45:18.519 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-10T22:45:18.519 INFO:teuthology.task.kernel:Updating grub on smithi053 to boot 4.18.0-552.el8.x86_64 2024-04-10T22:45:18.519 DEBUG:teuthology.orchestra.run.smithi053:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-10T22:45:19.867 INFO:teuthology.orchestra.run.smithi053.stderr:Generating grub configuration file ... 2024-04-10T22:45:21.759 INFO:teuthology.orchestra.run.smithi053.stderr:done 2024-04-10T22:45:21.770 DEBUG:teuthology.orchestra.run.smithi053:> mktemp 2024-04-10T22:45:21.798 INFO:teuthology.orchestra.run.smithi053.stdout:/tmp/tmp.FAidNo4P2U 2024-04-10T22:45:21.798 DEBUG:teuthology.orchestra.run.smithi053:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.FAidNo4P2U 2024-04-10T22:45:21.870 DEBUG:teuthology.orchestra.run.smithi053:> sudo chmod 0666 /tmp/tmp.FAidNo4P2U 2024-04-10T22:45:22.041 DEBUG:teuthology.orchestra.remote:smithi053:/tmp/tmp.FAidNo4P2U is 6KB 2024-04-10T22:45:22.089 DEBUG:teuthology.orchestra.run.smithi053:> rm -fr /tmp/tmp.FAidNo4P2U 2024-04-10T22:45:22.107 DEBUG:teuthology.orchestra.run.smithi053:> sudo /bin/ls /boot/loader/entries || true 2024-04-10T22:45:22.185 INFO:teuthology.orchestra.run.smithi053.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2024-04-10T22:45:22.185 INFO:teuthology.orchestra.run.smithi053.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2024-04-10T22:45:22.185 INFO:teuthology.orchestra.run.smithi053.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2024-04-10T22:45:22.185 INFO:teuthology.orchestra.run.smithi053.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64.conf 2024-04-10T22:45:22.186 DEBUG:teuthology.orchestra.run.smithi053:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64 2024-04-10T22:45:22.372 DEBUG:teuthology.orchestra.run.smithi053:> sudo shutdown -r now 2024-04-10T22:45:26.686 INFO:teuthology.orchestra.run.smithi031.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:45:26.711 INFO:teuthology.orchestra.run.smithi031.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:45:27.082 INFO:teuthology.orchestra.run.smithi031.stdout: Running scriptlet: kernel-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:45:27.083 INFO:teuthology.orchestra.run.smithi031.stdout: Verifying : kernel-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:45:27.083 INFO:teuthology.orchestra.run.smithi031.stdout: Verifying : kernel-core-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:45:27.584 INFO:teuthology.orchestra.run.smithi031.stdout: Verifying : kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:45:27.584 INFO:teuthology.orchestra.run.smithi031.stdout: 2024-04-10T22:45:27.584 INFO:teuthology.orchestra.run.smithi031.stdout:Installed: 2024-04-10T22:45:27.584 INFO:teuthology.orchestra.run.smithi031.stdout: kernel-4.18.0-552.el8.x86_64 kernel-core-4.18.0-552.el8.x86_64 2024-04-10T22:45:27.585 INFO:teuthology.orchestra.run.smithi031.stdout: kernel-modules-4.18.0-552.el8.x86_64 2024-04-10T22:45:27.585 INFO:teuthology.orchestra.run.smithi031.stdout: 2024-04-10T22:45:27.585 INFO:teuthology.orchestra.run.smithi031.stdout:Complete! 2024-04-10T22:45:27.722 DEBUG:teuthology.orchestra.run.smithi031:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:45:27.950 INFO:teuthology.orchestra.run.smithi031.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-10T22:45:27.950 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-10T22:45:27.950 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-552.el8.x86_64 2024-04-10T22:45:27.950 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-10T22:45:27.950 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-10T22:45:27.950 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-04-10T22:45:27.950 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi031.front.sepia.ceph.com, path=None, version=distro) 2024-04-10T22:45:27.950 DEBUG:teuthology.orchestra.run.smithi031:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:45:28.171 INFO:teuthology.orchestra.run.smithi031.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-10T22:45:28.171 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-10T22:45:28.171 DEBUG:teuthology.orchestra.run.smithi031:> sudo rpm -qi grub2-tools 2024-04-10T22:45:28.205 INFO:teuthology.orchestra.run.smithi031.stdout:Name : grub2-tools 2024-04-10T22:45:28.205 INFO:teuthology.orchestra.run.smithi031.stdout:Epoch : 1 2024-04-10T22:45:28.205 INFO:teuthology.orchestra.run.smithi031.stdout:Version : 2.02 2024-04-10T22:45:28.205 INFO:teuthology.orchestra.run.smithi031.stdout:Release : 129.el8 2024-04-10T22:45:28.206 INFO:teuthology.orchestra.run.smithi031.stdout:Architecture: x86_64 2024-04-10T22:45:28.206 INFO:teuthology.orchestra.run.smithi031.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2024-04-10T22:45:28.206 INFO:teuthology.orchestra.run.smithi031.stdout:Group : System Environment/Base 2024-04-10T22:45:28.206 INFO:teuthology.orchestra.run.smithi031.stdout:Size : 9265586 2024-04-10T22:45:28.206 INFO:teuthology.orchestra.run.smithi031.stdout:License : GPLv3+ 2024-04-10T22:45:28.206 INFO:teuthology.orchestra.run.smithi031.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2024-04-10T22:45:28.206 INFO:teuthology.orchestra.run.smithi031.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2024-04-10T22:45:28.206 INFO:teuthology.orchestra.run.smithi031.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2024-04-10T22:45:28.206 INFO:teuthology.orchestra.run.smithi031.stdout:Build Host : kbuilder.bsys.centos.org 2024-04-10T22:45:28.206 INFO:teuthology.orchestra.run.smithi031.stdout:Relocations : (not relocatable) 2024-04-10T22:45:28.206 INFO:teuthology.orchestra.run.smithi031.stdout:Packager : CentOS BuildSystem 2024-04-10T22:45:28.206 INFO:teuthology.orchestra.run.smithi031.stdout:Vendor : CentOS 2024-04-10T22:45:28.206 INFO:teuthology.orchestra.run.smithi031.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-10T22:45:28.206 INFO:teuthology.orchestra.run.smithi031.stdout:Summary : Support tools for GRUB. 2024-04-10T22:45:28.206 INFO:teuthology.orchestra.run.smithi031.stdout:Description : 2024-04-10T22:45:28.207 INFO:teuthology.orchestra.run.smithi031.stdout: 2024-04-10T22:45:28.207 INFO:teuthology.orchestra.run.smithi031.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-10T22:45:28.207 INFO:teuthology.orchestra.run.smithi031.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-10T22:45:28.207 INFO:teuthology.orchestra.run.smithi031.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-10T22:45:28.207 INFO:teuthology.orchestra.run.smithi031.stdout:hardware devices. 2024-04-10T22:45:28.207 INFO:teuthology.orchestra.run.smithi031.stdout: 2024-04-10T22:45:28.207 INFO:teuthology.orchestra.run.smithi031.stdout:This subpackage provides tools for support of all platforms. 2024-04-10T22:45:28.208 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-10T22:45:28.208 INFO:teuthology.task.kernel:Updating grub on smithi031 to boot 4.18.0-552.el8.x86_64 2024-04-10T22:45:28.208 DEBUG:teuthology.orchestra.run.smithi031:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-10T22:45:29.660 INFO:teuthology.orchestra.run.smithi031.stderr:Generating grub configuration file ... 2024-04-10T22:45:31.854 INFO:teuthology.orchestra.run.smithi031.stderr:done 2024-04-10T22:45:31.856 DEBUG:teuthology.orchestra.run.smithi031:> mktemp 2024-04-10T22:45:31.884 INFO:teuthology.orchestra.run.smithi031.stdout:/tmp/tmp.RLmpmL83zD 2024-04-10T22:45:31.884 DEBUG:teuthology.orchestra.run.smithi031:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.RLmpmL83zD 2024-04-10T22:45:31.956 DEBUG:teuthology.orchestra.run.smithi031:> sudo chmod 0666 /tmp/tmp.RLmpmL83zD 2024-04-10T22:45:32.115 DEBUG:teuthology.orchestra.remote:smithi031:/tmp/tmp.RLmpmL83zD is 6KB 2024-04-10T22:45:32.164 DEBUG:teuthology.orchestra.run.smithi031:> rm -fr /tmp/tmp.RLmpmL83zD 2024-04-10T22:45:32.182 DEBUG:teuthology.orchestra.run.smithi031:> sudo /bin/ls /boot/loader/entries || true 2024-04-10T22:45:32.261 INFO:teuthology.orchestra.run.smithi031.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2024-04-10T22:45:32.261 INFO:teuthology.orchestra.run.smithi031.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2024-04-10T22:45:32.261 INFO:teuthology.orchestra.run.smithi031.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2024-04-10T22:45:32.261 INFO:teuthology.orchestra.run.smithi031.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64.conf 2024-04-10T22:45:32.262 DEBUG:teuthology.orchestra.run.smithi031:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64 2024-04-10T22:45:32.409 DEBUG:teuthology.orchestra.run.smithi031:> sudo shutdown -r now 2024-04-10T22:45:52.393 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-04-10T22:45:52.394 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi053.front.sepia.ceph.com' 2024-04-10T22:45:52.395 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi053.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:46:02.418 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-04-10T22:46:02.418 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi031.front.sepia.ceph.com' 2024-04-10T22:46:02.419 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi031.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:46:52.475 DEBUG:teuthology.orchestra.remote:timed out 2024-04-10T22:47:01.485 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi053.front.sepia.ceph.com' 2024-04-10T22:47:01.487 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi053.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:47:01.728 DEBUG:teuthology.orchestra.run.smithi053:> true 2024-04-10T22:47:02.013 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi053.front.sepia.ceph.com' 2024-04-10T22:47:02.013 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "4.18.0-552.el8.x86_64"... 2024-04-10T22:47:02.013 DEBUG:teuthology.orchestra.run.smithi053:> uname -r 2024-04-10T22:47:02.033 INFO:teuthology.orchestra.run.smithi053.stdout:4.18.0-552.el8.x86_64 2024-04-10T22:47:02.034 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-552.el8.x86_64 vs 4.18.0-552.el8.x86_64 2024-04-10T22:47:02.034 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-10T22:47:02.034 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-10T22:47:02.451 DEBUG:teuthology.orchestra.remote:timed out 2024-04-10T22:47:03.035 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-04-10T22:47:03.036 DEBUG:teuthology.orchestra.run.smithi053:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-10T22:47:03.126 INFO:teuthology.orchestra.run.smithi053.stdout:ttyS1 2024-04-10T22:47:03.140 DEBUG:teuthology.parallel:result is None 2024-04-10T22:47:11.452 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi031.front.sepia.ceph.com' 2024-04-10T22:47:11.452 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi031.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:47:11.810 DEBUG:teuthology.orchestra.run.smithi031:> true 2024-04-10T22:47:12.088 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi031.front.sepia.ceph.com' 2024-04-10T22:47:12.088 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "4.18.0-552.el8.x86_64"... 2024-04-10T22:47:12.088 DEBUG:teuthology.orchestra.run.smithi031:> uname -r 2024-04-10T22:47:12.108 INFO:teuthology.orchestra.run.smithi031.stdout:4.18.0-552.el8.x86_64 2024-04-10T22:47:12.108 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-552.el8.x86_64 vs 4.18.0-552.el8.x86_64 2024-04-10T22:47:12.108 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-10T22:47:12.108 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-10T22:47:13.109 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-04-10T22:47:13.109 DEBUG:teuthology.orchestra.run.smithi031:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-10T22:47:13.183 INFO:teuthology.orchestra.run.smithi031.stdout:ttyS1 2024-04-10T22:47:13.197 DEBUG:teuthology.parallel:result is None 2024-04-10T22:47:13.197 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-10T22:47:13.205 INFO:teuthology.task.internal:Creating test directory... 2024-04-10T22:47:13.205 DEBUG:teuthology.orchestra.run.smithi031:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-10T22:47:13.208 DEBUG:teuthology.orchestra.run.smithi053:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-10T22:47:13.230 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-10T22:47:13.237 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-10T22:47:13.244 INFO:teuthology.task.internal:Creating archive directory... 2024-04-10T22:47:13.244 DEBUG:teuthology.orchestra.run.smithi031:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-10T22:47:13.270 DEBUG:teuthology.orchestra.run.smithi053:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-10T22:47:13.314 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-10T22:47:13.321 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-10T22:47:13.321 DEBUG:teuthology.orchestra.run.smithi031:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-10T22:47:13.344 DEBUG:teuthology.orchestra.run.smithi053:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-10T22:47:13.383 INFO:teuthology.orchestra.run.smithi031.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:47:13.393 INFO:teuthology.orchestra.run.smithi053.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:47:13.395 INFO:teuthology.orchestra.run.smithi031.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:47:13.402 INFO:teuthology.orchestra.run.smithi053.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:47:13.404 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-10T22:47:13.411 INFO:teuthology.task.internal:Configuring sudo... 2024-04-10T22:47:13.411 DEBUG:teuthology.orchestra.run.smithi031:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-10T22:47:13.438 DEBUG:teuthology.orchestra.run.smithi053:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-10T22:47:13.478 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-10T22:47:13.488 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-10T22:47:13.488 DEBUG:teuthology.orchestra.run.smithi031:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-10T22:47:13.509 DEBUG:teuthology.orchestra.run.smithi053:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-10T22:47:13.539 DEBUG:teuthology.orchestra.run.smithi031:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:47:13.610 DEBUG:teuthology.orchestra.run.smithi031:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:47:13.661 DEBUG:teuthology.orchestra.run.smithi031:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:47:13.749 DEBUG:teuthology.orchestra.run.smithi031:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:47:13.791 DEBUG:teuthology.orchestra.run.smithi031:> set -ex 2024-04-10T22:47:13.791 DEBUG:teuthology.orchestra.run.smithi031:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-10T22:47:13.859 DEBUG:teuthology.orchestra.run.smithi053:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:47:13.909 DEBUG:teuthology.orchestra.run.smithi053:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:47:13.966 DEBUG:teuthology.orchestra.run.smithi053:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:47:14.054 DEBUG:teuthology.orchestra.run.smithi053:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:47:14.100 DEBUG:teuthology.orchestra.run.smithi053:> set -ex 2024-04-10T22:47:14.101 DEBUG:teuthology.orchestra.run.smithi053:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-10T22:47:14.171 DEBUG:teuthology.orchestra.run.smithi031:> sudo service rsyslog restart 2024-04-10T22:47:14.173 DEBUG:teuthology.orchestra.run.smithi053:> sudo service rsyslog restart 2024-04-10T22:47:14.221 INFO:teuthology.orchestra.run.smithi031.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:47:14.272 INFO:teuthology.orchestra.run.smithi053.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:47:14.752 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-10T22:47:14.762 INFO:teuthology.task.internal:Starting timer... 2024-04-10T22:47:14.762 INFO:teuthology.run_tasks:Running task pcp... 2024-04-10T22:47:14.771 INFO:teuthology.run_tasks:Running task selinux... 2024-04-10T22:47:14.780 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-04-10T22:47:14.781 DEBUG:teuthology.orchestra.run.smithi031:> sudo service auditd rotate 2024-04-10T22:47:14.882 INFO:teuthology.orchestra.run.smithi031.stdout:Rotating logs: 2024-04-10T22:47:14.884 DEBUG:teuthology.orchestra.run.smithi053:> sudo service auditd rotate 2024-04-10T22:47:14.954 INFO:teuthology.orchestra.run.smithi053.stdout:Rotating logs: 2024-04-10T22:47:14.956 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-10T22:47:14.956 DEBUG:teuthology.orchestra.run.smithi031:> /usr/sbin/getenforce 2024-04-10T22:47:14.991 INFO:teuthology.orchestra.run.smithi031.stdout:Permissive 2024-04-10T22:47:14.991 DEBUG:teuthology.orchestra.run.smithi053:> /usr/sbin/getenforce 2024-04-10T22:47:15.043 INFO:teuthology.orchestra.run.smithi053.stdout:Permissive 2024-04-10T22:47:15.043 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi031.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi053.front.sepia.ceph.com': 'permissive'} 2024-04-10T22:47:15.044 DEBUG:teuthology.orchestra.run.smithi031:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:47:15.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:47:15.073 DEBUG:teuthology.orchestra.run.smithi053:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:47:15.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:47:15.104 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-10T22:47:15.104 DEBUG:teuthology.orchestra.run.smithi031:> sudo /usr/sbin/setenforce permissive 2024-04-10T22:47:15.142 DEBUG:teuthology.orchestra.run.smithi053:> sudo /usr/sbin/setenforce permissive 2024-04-10T22:47:15.180 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-10T22:47:15.190 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-10T22:47:15.208 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-10T22:47:15.210 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi031.front.sepia.ceph.com,smithi053.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-10T23:02:20.311 INFO:teuthology.task.ansible:Archiving ansible failure log at: /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650198/ansible_failures.yaml 2024-04-10T23:02:20.314 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/__init__.py", line 123, in __enter__ self.begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 508, in begin super(CephLab, self).begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 342, in begin self.execute_playbook() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 374, in execute_playbook self._handle_failure(command, status) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 402, in _handle_failure raise AnsibleFailedError(failures) teuthology.exceptions.AnsibleFailedError: [',', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at'] 2024-04-10T23:02:20.515 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=11a497216e97411d8bb5a35cd5f5e4ca Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/__init__.py", line 123, in __enter__ self.begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 508, in begin super(CephLab, self).begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 342, in begin self.execute_playbook() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 374, in execute_playbook self._handle_failure(command, status) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 402, in _handle_failure raise AnsibleFailedError(failures) teuthology.exceptions.AnsibleFailedError: [',', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at'] 2024-04-10T23:02:20.517 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-10T23:02:20.525 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-10T23:02:20.525 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-10T23:02:20.541 DEBUG:teuthology.orchestra.run.smithi031:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-10T23:02:20.626 DEBUG:teuthology.orchestra.run.smithi053:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-10T23:02:20.699 DEBUG:teuthology.orchestra.run.smithi031:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T23:02:20.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T23:02:20.733 DEBUG:teuthology.orchestra.run.smithi053:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T23:02:20.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T23:02:20.767 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-10T23:02:20.775 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-10T23:02:20.783 INFO:teuthology.task.internal:Duration was 906.021356 seconds 2024-04-10T23:02:20.783 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-10T23:02:20.791 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-10T23:02:20.791 DEBUG:teuthology.orchestra.run.smithi031:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-10T23:02:20.793 DEBUG:teuthology.orchestra.run.smithi053:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-10T23:02:20.843 INFO:teuthology.orchestra.run.smithi031.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T23:02:20.856 INFO:teuthology.orchestra.run.smithi053.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T23:02:21.143 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-10T23:02:21.143 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi031.front.sepia.ceph.com 2024-04-10T23:02:21.143 DEBUG:teuthology.orchestra.run.smithi031:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-10T23:02:21.168 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi053.front.sepia.ceph.com 2024-04-10T23:02:21.168 DEBUG:teuthology.orchestra.run.smithi053:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-10T23:02:21.193 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-10T23:02:21.194 DEBUG:teuthology.orchestra.run.smithi031:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-10T23:02:21.211 DEBUG:teuthology.orchestra.run.smithi053:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-10T23:02:21.277 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-10T23:02:21.277 DEBUG:teuthology.orchestra.run.smithi031:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-10T23:02:21.294 DEBUG:teuthology.orchestra.run.smithi053:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-10T23:02:21.382 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-10T23:02:21.391 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-10T23:02:21.391 DEBUG:teuthology.orchestra.run.smithi031:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-10T23:02:21.419 DEBUG:teuthology.orchestra.run.smithi053:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-10T23:02:21.447 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-10T23:02:21.454 DEBUG:teuthology.orchestra.run.smithi031:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-10T23:02:21.462 DEBUG:teuthology.orchestra.run.smithi053:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-10T23:02:21.486 INFO:teuthology.orchestra.run.smithi031.stdout:kernel.core_pattern = core 2024-04-10T23:02:21.515 INFO:teuthology.orchestra.run.smithi053.stdout:kernel.core_pattern = core 2024-04-10T23:02:21.539 DEBUG:teuthology.orchestra.run.smithi031:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-10T23:02:21.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T23:02:21.566 DEBUG:teuthology.orchestra.run.smithi053:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-10T23:02:21.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T23:02:21.597 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-10T23:02:21.606 INFO:teuthology.task.internal:Transferring archived files... 2024-04-10T23:02:21.608 DEBUG:teuthology.misc:Transferring archived files from smithi031:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650198/remote/smithi031 2024-04-10T23:02:21.609 DEBUG:teuthology.orchestra.run.smithi031:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-10T23:02:21.657 DEBUG:teuthology.misc:Transferring archived files from smithi053:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650198/remote/smithi053 2024-04-10T23:02:21.658 DEBUG:teuthology.orchestra.run.smithi053:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-10T23:02:21.714 INFO:teuthology.task.internal:Removing archive directory... 2024-04-10T23:02:21.715 DEBUG:teuthology.orchestra.run.smithi031:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-10T23:02:21.717 DEBUG:teuthology.orchestra.run.smithi053:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-10T23:02:21.734 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-10T23:02:21.744 INFO:teuthology.task.internal:Not uploading archives. 2024-04-10T23:02:21.744 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-10T23:02:21.790 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-10T23:02:21.790 DEBUG:teuthology.orchestra.run.smithi031:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-10T23:02:21.793 DEBUG:teuthology.orchestra.run.smithi053:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-10T23:02:21.809 INFO:teuthology.orchestra.run.smithi031.stdout: 525355 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 10 23:02 /home/ubuntu/cephtest 2024-04-10T23:02:21.810 INFO:teuthology.orchestra.run.smithi053.stdout: 393225 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 10 23:02 /home/ubuntu/cephtest 2024-04-10T23:02:21.812 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-10T23:02:21.821 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-10T23:02:21.871 INFO:teuthology.nuke:Checking targets against current locks 2024-04-10T23:02:21.907 DEBUG:teuthology.nuke:shortname: smithi031 2024-04-10T23:02:21.907 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T23:02:21.953 DEBUG:teuthology.nuke:shortname: smithi053 2024-04-10T23:02:21.953 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T23:02:21.968 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi031.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650198', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:34:33.358071', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNKFMx3gdxQeHDIKftIK2KVWZQ5YFl0fQtYGhwCVyPNgZfopwhxtBUWQ3yTgCqnUKvSRAn0Tuw5VsUaKo4lTCm4='} 2024-04-10T23:02:22.004 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi053.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650198', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:34:33.356032', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFScDMQ5OzYa1CXENdMQJKk1XXUxkoFu6bwIrp3FWsiGFOFGwB8v4DpPrFT7LHPgPfgNtZUZuIQlbT3CmaGRyE0='} 2024-04-10T23:02:22.056 INFO:teuthology.orchestra.console.smithi031:Power off 2024-04-10T23:02:22.057 DEBUG:teuthology.orchestra.console.smithi031:pexpect command: ipmitool -H smithi031.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-10T23:02:22.113 INFO:teuthology.orchestra.console.smithi053:Power off 2024-04-10T23:02:22.113 DEBUG:teuthology.orchestra.console.smithi053:pexpect command: ipmitool -H smithi053.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-10T23:02:22.123 DEBUG:teuthology.orchestra.console.smithi031:power off output: Chassis Power Control: Down/Off 2024-04-10T23:02:22.123 DEBUG:teuthology.orchestra.console.smithi031:pexpect command: ipmitool -H smithi031.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:02:22.177 DEBUG:teuthology.orchestra.console.smithi053:power off output: Chassis Power Control: Down/Off 2024-04-10T23:02:22.178 DEBUG:teuthology.orchestra.console.smithi053:pexpect command: ipmitool -H smithi053.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:02:22.188 DEBUG:teuthology.orchestra.console.smithi031:check power output: Chassis Power is on 2024-04-10T23:02:22.230 DEBUG:teuthology.orchestra.console.smithi053:check power output: Chassis Power is on 2024-04-10T23:02:26.189 DEBUG:teuthology.orchestra.console.smithi031:pexpect command: ipmitool -H smithi031.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:02:26.232 DEBUG:teuthology.orchestra.console.smithi053:pexpect command: ipmitool -H smithi053.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:02:26.300 DEBUG:teuthology.orchestra.console.smithi031:check power output: Chassis Power is on 2024-04-10T23:02:26.341 DEBUG:teuthology.orchestra.console.smithi053:check power output: Chassis Power is on 2024-04-10T23:02:30.302 DEBUG:teuthology.orchestra.console.smithi031:pexpect command: ipmitool -H smithi031.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:02:30.342 DEBUG:teuthology.orchestra.console.smithi053:pexpect command: ipmitool -H smithi053.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:02:30.413 DEBUG:teuthology.orchestra.console.smithi031:check power output: Chassis Power is on 2024-04-10T23:02:30.453 DEBUG:teuthology.orchestra.console.smithi053:check power output: Chassis Power is off 2024-04-10T23:02:30.553 INFO:teuthology.orchestra.console.smithi053:Power off completed 2024-04-10T23:02:34.416 DEBUG:teuthology.orchestra.console.smithi031:pexpect command: ipmitool -H smithi031.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:02:34.527 DEBUG:teuthology.orchestra.console.smithi031:check power output: Chassis Power is off 2024-04-10T23:02:34.627 INFO:teuthology.orchestra.console.smithi031:Power off completed 2024-04-10T23:02:34.733 INFO:teuthology.run:Summary data: description: upgrade:pacific-x/stress-split/{0-distro/centos_8.stream_container_tools 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} duration: 906.021356344223 failure_reason: '['','', ''Bugzilla. In case of problems, contact the owner of this repository.'', ''Enabling a Copr repository. Please note that this repository is not part'', ''Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable'', ''Please do not file bug reports about these packages in Fedora'', ''The Fedora Project does not exercise any power over the contents of'', ''and packages are not held to any quality or security level.'', ''of the main distribution, and quality may vary.'', ''this repository beyond the rules outlined in the Copr FAQ at'']' owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=11a497216e97411d8bb5a35cd5f5e4ca status: dead success: false 2024-04-10T23:02:34.734 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-10T23:02:34.818 INFO:teuthology.run:DEAD