2024-04-10T22:37:30.220 INFO:root:teuthology version: 0.0.1.dev267+g6c63784 2024-04-10T22:37:30.220 DEBUG:teuthology.run:Teuthology command: teuthology --owner scheduled_teuthology@teuthology --verbose --description upgrade:pacific-x/stress-split/{0-distro/centos_8.stream_container_tools 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} --name teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi --archive /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650183 -- /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650183/orig.config.yaml 2024-04-10T22:37:30.274 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-10T22:37:30.367 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650183 branch: reef description: upgrade:pacific-x/stress-split/{0-distro/centos_8.stream_container_tools 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} email: ceph-qa@ceph.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: pacific clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' first_in_suite: false job_id: '7650183' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'librbd C and C++ api tests ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run basic import/export cli tests for rbd ' name: teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 8.stream overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: reef sha1: 4f70a9a90881107996c8b7a3e89ba8b32624713c owner: scheduled_teuthology@teuthology priority: 830 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' seed: 4604 sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: pacific clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 0/1 suite: upgrade:pacific-x suite_branch: reef suite_path: /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 4f70a9a90881107996c8b7a3e89ba8b32624713c targets: smithi120.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDORNFmvtZrQHmn/4/z3NftYfW4emO0WurdXs4mBiQhR6laI7LK10k6Z3ywA4IF7ZvsU/EnwWLswiOmZXAty/d0= smithi151.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCQEu8BXZEJ6hwD8DeLEyN6GMDvjUgnEGx3k5IRicfbZEbNaXMzn2ELP+N9c04VI4mcDHA7O2+7+JlP/QZFyDAE= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - install: branch: pacific exclude_packages: - ceph-volume - cephadm: cephadm_branch: pacific cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:pacific - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 6c637841c215537a4502385240412f1966e0faab timestamp: 2024-04-10_01:08:08 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-04-10T22:37:30.368 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa; will attempt to use it 2024-04-10T22:37:30.368 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa/tasks 2024-04-10T22:37:30.368 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-10T22:37:30.370 INFO:teuthology.task.internal:Checking packages... 2024-04-10T22:37:30.392 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'd540ebaca6b131a1dd560e7f69e024b133bbaa42' 2024-04-10T22:37:30.392 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-10T22:37:30.393 INFO:teuthology.packaging:ref: None 2024-04-10T22:37:30.393 INFO:teuthology.packaging:tag: None 2024-04-10T22:37:30.393 INFO:teuthology.packaging:branch: reef 2024-04-10T22:37:30.393 INFO:teuthology.packaging:sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 2024-04-10T22:37:30.393 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef 2024-04-10T22:37:30.761 INFO:teuthology.task.internal:Found packages for ceph version 18.2.2-1189.gc96c8d47 2024-04-10T22:37:30.762 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-10T22:37:30.769 INFO:teuthology.task.internal:no buildpackages task found 2024-04-10T22:37:30.769 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-10T22:37:30.775 INFO:teuthology.task.internal:Saving configuration 2024-04-10T22:37:30.796 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-10T22:37:30.802 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T22:37:30.824 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi120.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650183', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:16.683421', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDORNFmvtZrQHmn/4/z3NftYfW4emO0WurdXs4mBiQhR6laI7LK10k6Z3ywA4IF7ZvsU/EnwWLswiOmZXAty/d0='} 2024-04-10T22:37:30.846 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi151.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650183', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:16.687307', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCQEu8BXZEJ6hwD8DeLEyN6GMDvjUgnEGx3k5IRicfbZEbNaXMzn2ELP+N9c04VI4mcDHA7O2+7+JlP/QZFyDAE='} 2024-04-10T22:37:30.846 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-10T22:37:30.852 INFO:teuthology.task.internal:roles: ubuntu@smithi120.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-04-10T22:37:30.852 INFO:teuthology.task.internal:roles: ubuntu@smithi151.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-04-10T22:37:30.852 INFO:teuthology.run_tasks:Running task console_log... 2024-04-10T22:37:30.921 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f836f9400d0>, signals=[15]) 2024-04-10T22:37:30.922 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-10T22:37:30.928 INFO:teuthology.task.internal:Opening connections... 2024-04-10T22:37:30.928 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi120.front.sepia.ceph.com 2024-04-10T22:37:30.930 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi120.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:37:31.008 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi151.front.sepia.ceph.com 2024-04-10T22:37:31.009 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi151.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:37:31.082 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-10T22:37:31.089 DEBUG:teuthology.orchestra.run.smithi120:> uname -m 2024-04-10T22:37:31.112 INFO:teuthology.orchestra.run.smithi120.stdout:x86_64 2024-04-10T22:37:31.112 DEBUG:teuthology.orchestra.run.smithi120:> cat /etc/os-release 2024-04-10T22:37:31.173 INFO:teuthology.orchestra.run.smithi120.stdout:NAME="CentOS Stream" 2024-04-10T22:37:31.173 INFO:teuthology.orchestra.run.smithi120.stdout:VERSION="8" 2024-04-10T22:37:31.173 INFO:teuthology.orchestra.run.smithi120.stdout:ID="centos" 2024-04-10T22:37:31.173 INFO:teuthology.orchestra.run.smithi120.stdout:ID_LIKE="rhel fedora" 2024-04-10T22:37:31.173 INFO:teuthology.orchestra.run.smithi120.stdout:VERSION_ID="8" 2024-04-10T22:37:31.173 INFO:teuthology.orchestra.run.smithi120.stdout:PLATFORM_ID="platform:el8" 2024-04-10T22:37:31.173 INFO:teuthology.orchestra.run.smithi120.stdout:PRETTY_NAME="CentOS Stream 8" 2024-04-10T22:37:31.173 INFO:teuthology.orchestra.run.smithi120.stdout:ANSI_COLOR="0;31" 2024-04-10T22:37:31.173 INFO:teuthology.orchestra.run.smithi120.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2024-04-10T22:37:31.173 INFO:teuthology.orchestra.run.smithi120.stdout:HOME_URL="https://centos.org/" 2024-04-10T22:37:31.173 INFO:teuthology.orchestra.run.smithi120.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-10T22:37:31.173 INFO:teuthology.orchestra.run.smithi120.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2024-04-10T22:37:31.173 INFO:teuthology.orchestra.run.smithi120.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-10T22:37:31.174 INFO:teuthology.lock.ops:Updating smithi120.front.sepia.ceph.com on lock server 2024-04-10T22:37:31.194 DEBUG:teuthology.orchestra.run.smithi151:> uname -m 2024-04-10T22:37:31.217 INFO:teuthology.orchestra.run.smithi151.stdout:x86_64 2024-04-10T22:37:31.217 DEBUG:teuthology.orchestra.run.smithi151:> cat /etc/os-release 2024-04-10T22:37:31.277 INFO:teuthology.orchestra.run.smithi151.stdout:NAME="CentOS Stream" 2024-04-10T22:37:31.278 INFO:teuthology.orchestra.run.smithi151.stdout:VERSION="8" 2024-04-10T22:37:31.278 INFO:teuthology.orchestra.run.smithi151.stdout:ID="centos" 2024-04-10T22:37:31.278 INFO:teuthology.orchestra.run.smithi151.stdout:ID_LIKE="rhel fedora" 2024-04-10T22:37:31.278 INFO:teuthology.orchestra.run.smithi151.stdout:VERSION_ID="8" 2024-04-10T22:37:31.278 INFO:teuthology.orchestra.run.smithi151.stdout:PLATFORM_ID="platform:el8" 2024-04-10T22:37:31.278 INFO:teuthology.orchestra.run.smithi151.stdout:PRETTY_NAME="CentOS Stream 8" 2024-04-10T22:37:31.278 INFO:teuthology.orchestra.run.smithi151.stdout:ANSI_COLOR="0;31" 2024-04-10T22:37:31.278 INFO:teuthology.orchestra.run.smithi151.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2024-04-10T22:37:31.278 INFO:teuthology.orchestra.run.smithi151.stdout:HOME_URL="https://centos.org/" 2024-04-10T22:37:31.278 INFO:teuthology.orchestra.run.smithi151.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-10T22:37:31.278 INFO:teuthology.orchestra.run.smithi151.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2024-04-10T22:37:31.278 INFO:teuthology.orchestra.run.smithi151.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-10T22:37:31.279 INFO:teuthology.lock.ops:Updating smithi151.front.sepia.ceph.com on lock server 2024-04-10T22:37:31.302 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-10T22:37:31.311 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-10T22:37:31.412 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-10T22:37:31.412 DEBUG:teuthology.orchestra.run.smithi120:> test '!' -e /home/ubuntu/cephtest 2024-04-10T22:37:31.414 DEBUG:teuthology.orchestra.run.smithi151:> test '!' -e /home/ubuntu/cephtest 2024-04-10T22:37:31.435 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-10T22:37:31.441 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-10T22:37:31.441 DEBUG:teuthology.orchestra.run.smithi120:> test -z $(ls -A /var/lib/ceph) 2024-04-10T22:37:31.476 DEBUG:teuthology.orchestra.run.smithi151:> test -z $(ls -A /var/lib/ceph) 2024-04-10T22:37:31.522 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-10T22:37:31.579 INFO:teuthology.run_tasks:Running task kernel... 2024-04-10T22:37:31.595 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-04-10T22:37:31.595 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-04-10T22:37:31.596 DEBUG:teuthology.orchestra.run.smithi120:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-10T22:37:31.596 DEBUG:teuthology.orchestra.run.smithi151:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-10T22:37:31.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:37:31.617 DEBUG:teuthology.orchestra.run.smithi120:> uname -r 2024-04-10T22:37:31.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:37:31.618 DEBUG:teuthology.orchestra.run.smithi151:> uname -r 2024-04-10T22:37:31.678 INFO:teuthology.orchestra.run.smithi120.stdout:4.18.0-489.el8.x86_64 2024-04-10T22:37:31.678 INFO:teuthology.task.kernel:Running kernel on smithi120: 4.18.0-489.el8.x86_64 2024-04-10T22:37:31.678 DEBUG:teuthology.orchestra.run.smithi120:> sudo yum install -y kernel 2024-04-10T22:37:31.679 INFO:teuthology.orchestra.run.smithi151.stdout:4.18.0-489.el8.x86_64 2024-04-10T22:37:31.679 INFO:teuthology.task.kernel:Running kernel on smithi151: 4.18.0-489.el8.x86_64 2024-04-10T22:37:31.680 DEBUG:teuthology.orchestra.run.smithi151:> sudo yum install -y kernel 2024-04-10T22:37:35.494 INFO:teuthology.orchestra.run.smithi120.stdout:CentOS Stream 8 - AppStream 24 MB/s | 28 MB 00:01 2024-04-10T22:37:41.581 INFO:teuthology.orchestra.run.smithi120.stdout:CentOS Stream 8 - BaseOS 10 MB/s | 10 MB 00:01 2024-04-10T22:39:47.705 INFO:teuthology.orchestra.run.smithi151.stdout:CentOS Stream 8 - AppStream 214 kB/s | 28 MB 02:13 2024-04-10T22:39:52.419 INFO:teuthology.orchestra.run.smithi120.stdout:CentOS Stream 8 - Extras 141 B/s | 18 kB 02:09 2024-04-10T22:39:52.870 INFO:teuthology.orchestra.run.smithi120.stdout:CentOS Stream 8 - Extras common packages 23 kB/s | 7.7 kB 00:00 2024-04-10T22:39:53.705 INFO:teuthology.orchestra.run.smithi151.stdout:CentOS Stream 8 - BaseOS 11 MB/s | 10 MB 00:00 2024-04-10T22:40:35.855 INFO:teuthology.orchestra.run.smithi120.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:42 2024-04-10T22:40:35.856 INFO:teuthology.orchestra.run.smithi120.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:40:35.856 INFO:teuthology.orchestra.run.smithi120.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:40:35.856 INFO:teuthology.orchestra.run.smithi120.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:42:06.613 INFO:teuthology.orchestra.run.smithi151.stdout:CentOS Stream 8 - Extras 139 B/s | 18 kB 02:11 2024-04-10T22:42:07.071 INFO:teuthology.orchestra.run.smithi151.stdout:CentOS Stream 8 - Extras common packages 22 kB/s | 7.7 kB 00:00 2024-04-10T22:42:47.255 INFO:teuthology.orchestra.run.smithi151.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:40 2024-04-10T22:42:47.256 INFO:teuthology.orchestra.run.smithi151.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:42:47.256 INFO:teuthology.orchestra.run.smithi151.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:42:47.256 INFO:teuthology.orchestra.run.smithi151.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:42:50.389 INFO:teuthology.orchestra.run.smithi120.stdout:Extra Packages for Enterprise Linux 124 kB/s | 16 MB 02:14 2024-04-10T22:42:54.189 INFO:teuthology.orchestra.run.smithi120.stdout:lab-extras 330 kB/s | 24 kB 00:00 2024-04-10T22:42:54.330 INFO:teuthology.orchestra.run.smithi120.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:42:57.102 INFO:teuthology.orchestra.run.smithi120.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2024-04-10T22:42:57.102 INFO:teuthology.orchestra.run.smithi120.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2024-04-10T22:42:57.232 INFO:teuthology.orchestra.run.smithi120.stdout:Dependencies resolved. 2024-04-10T22:42:57.233 INFO:teuthology.orchestra.run.smithi120.stdout:================================================================================ 2024-04-10T22:42:57.233 INFO:teuthology.orchestra.run.smithi120.stdout: Package Architecture Version Repository Size 2024-04-10T22:42:57.233 INFO:teuthology.orchestra.run.smithi120.stdout:================================================================================ 2024-04-10T22:42:57.233 INFO:teuthology.orchestra.run.smithi120.stdout:Installing: 2024-04-10T22:42:57.234 INFO:teuthology.orchestra.run.smithi120.stdout: kernel x86_64 4.18.0-552.el8 baseos 10 M 2024-04-10T22:42:57.234 INFO:teuthology.orchestra.run.smithi120.stdout:Installing dependencies: 2024-04-10T22:42:57.234 INFO:teuthology.orchestra.run.smithi120.stdout: kernel-core x86_64 4.18.0-552.el8 baseos 44 M 2024-04-10T22:42:57.234 INFO:teuthology.orchestra.run.smithi120.stdout: kernel-modules x86_64 4.18.0-552.el8 baseos 36 M 2024-04-10T22:42:57.234 INFO:teuthology.orchestra.run.smithi120.stdout: 2024-04-10T22:42:57.234 INFO:teuthology.orchestra.run.smithi120.stdout:Transaction Summary 2024-04-10T22:42:57.234 INFO:teuthology.orchestra.run.smithi120.stdout:================================================================================ 2024-04-10T22:42:57.234 INFO:teuthology.orchestra.run.smithi120.stdout:Install 3 Packages 2024-04-10T22:42:57.234 INFO:teuthology.orchestra.run.smithi120.stdout: 2024-04-10T22:42:57.235 INFO:teuthology.orchestra.run.smithi120.stdout:Total download size: 90 M 2024-04-10T22:42:57.235 INFO:teuthology.orchestra.run.smithi120.stdout:Installed size: 96 M 2024-04-10T22:42:57.235 INFO:teuthology.orchestra.run.smithi120.stdout:Downloading Packages: 2024-04-10T22:44:59.458 INFO:teuthology.orchestra.run.smithi151.stdout:Extra Packages for Enterprise Linux 126 kB/s | 16 MB 02:12 2024-04-10T22:45:03.207 INFO:teuthology.orchestra.run.smithi151.stdout:lab-extras 499 kB/s | 24 kB 00:00 2024-04-10T22:45:03.354 INFO:teuthology.orchestra.run.smithi151.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:45:06.146 INFO:teuthology.orchestra.run.smithi151.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2024-04-10T22:45:06.146 INFO:teuthology.orchestra.run.smithi151.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2024-04-10T22:45:06.270 INFO:teuthology.orchestra.run.smithi151.stdout:Dependencies resolved. 2024-04-10T22:45:06.271 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-10T22:45:06.271 INFO:teuthology.orchestra.run.smithi151.stdout: Package Architecture Version Repository Size 2024-04-10T22:45:06.271 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-10T22:45:06.271 INFO:teuthology.orchestra.run.smithi151.stdout:Installing: 2024-04-10T22:45:06.271 INFO:teuthology.orchestra.run.smithi151.stdout: kernel x86_64 4.18.0-552.el8 baseos 10 M 2024-04-10T22:45:06.271 INFO:teuthology.orchestra.run.smithi151.stdout:Installing dependencies: 2024-04-10T22:45:06.271 INFO:teuthology.orchestra.run.smithi151.stdout: kernel-core x86_64 4.18.0-552.el8 baseos 44 M 2024-04-10T22:45:06.271 INFO:teuthology.orchestra.run.smithi151.stdout: kernel-modules x86_64 4.18.0-552.el8 baseos 36 M 2024-04-10T22:45:06.272 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-10T22:45:06.272 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction Summary 2024-04-10T22:45:06.272 INFO:teuthology.orchestra.run.smithi151.stdout:================================================================================ 2024-04-10T22:45:06.272 INFO:teuthology.orchestra.run.smithi151.stdout:Install 3 Packages 2024-04-10T22:45:06.272 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-10T22:45:06.272 INFO:teuthology.orchestra.run.smithi151.stdout:Total download size: 90 M 2024-04-10T22:45:06.272 INFO:teuthology.orchestra.run.smithi151.stdout:Installed size: 96 M 2024-04-10T22:45:06.273 INFO:teuthology.orchestra.run.smithi151.stdout:Downloading Packages: 2024-04-10T22:45:08.219 INFO:teuthology.orchestra.run.smithi151.stdout:(1/3): kernel-4.18.0-552.el8.x86_64.rpm 6.1 MB/s | 10 MB 00:01 2024-04-10T22:45:09.356 INFO:teuthology.orchestra.run.smithi120.stdout:(1/3): kernel-4.18.0-552.el8.x86_64.rpm 5.7 MB/s | 10 MB 00:01 2024-04-10T22:45:23.437 INFO:teuthology.orchestra.run.smithi120.stdout:(2/3): kernel-core-4.18.0-552.el8.x86_64.rpm 2.7 MB/s | 44 MB 00:15 2024-04-10T22:45:24.445 INFO:teuthology.orchestra.run.smithi120.stdout:(3/3): kernel-modules-4.18.0-552.el8.x86_64.rpm 2.2 MB/s | 36 MB 00:16 2024-04-10T22:45:24.446 INFO:teuthology.orchestra.run.smithi120.stdout:-------------------------------------------------------------------------------- 2024-04-10T22:45:24.446 INFO:teuthology.orchestra.run.smithi120.stdout:Total 629 kB/s | 90 MB 02:27 2024-04-10T22:45:25.124 INFO:teuthology.orchestra.run.smithi120.stdout:Running transaction check 2024-04-10T22:45:25.556 INFO:teuthology.orchestra.run.smithi120.stdout:Transaction check succeeded. 2024-04-10T22:45:25.556 INFO:teuthology.orchestra.run.smithi120.stdout:Running transaction test 2024-04-10T22:45:26.817 INFO:teuthology.orchestra.run.smithi120.stdout:Transaction test succeeded. 2024-04-10T22:45:26.847 INFO:teuthology.orchestra.run.smithi120.stdout:Running transaction 2024-04-10T22:45:27.123 INFO:teuthology.orchestra.run.smithi151.stdout:(2/3): kernel-modules-4.18.0-552.el8.x86_64.rpm 1.8 MB/s | 36 MB 00:20 2024-04-10T22:45:28.282 INFO:teuthology.orchestra.run.smithi151.stdout:(3/3): kernel-core-4.18.0-552.el8.x86_64.rpm 2.0 MB/s | 44 MB 00:21 2024-04-10T22:45:28.282 INFO:teuthology.orchestra.run.smithi151.stdout:-------------------------------------------------------------------------------- 2024-04-10T22:45:28.282 INFO:teuthology.orchestra.run.smithi151.stdout:Total 4.1 MB/s | 90 MB 00:22 2024-04-10T22:45:28.965 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction check 2024-04-10T22:45:29.397 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction check succeeded. 2024-04-10T22:45:29.397 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction test 2024-04-10T22:45:29.683 INFO:teuthology.orchestra.run.smithi120.stdout: Preparing : 1/1 2024-04-10T22:45:30.285 INFO:teuthology.orchestra.run.smithi120.stdout: Installing : kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:45:30.671 INFO:teuthology.orchestra.run.smithi151.stdout:Transaction test succeeded. 2024-04-10T22:45:30.713 INFO:teuthology.orchestra.run.smithi151.stdout:Running transaction 2024-04-10T22:45:33.560 INFO:teuthology.orchestra.run.smithi151.stdout: Preparing : 1/1 2024-04-10T22:45:33.853 INFO:teuthology.orchestra.run.smithi120.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:45:34.052 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:45:34.276 INFO:teuthology.orchestra.run.smithi120.stdout: Installing : kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:45:37.661 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:45:37.729 INFO:teuthology.orchestra.run.smithi120.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:45:38.008 INFO:teuthology.orchestra.run.smithi120.stdout: Installing : kernel-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:45:38.044 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:45:41.501 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:45:41.808 INFO:teuthology.orchestra.run.smithi151.stdout: Installing : kernel-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:45:59.833 INFO:teuthology.orchestra.run.smithi120.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:45:59.858 INFO:teuthology.orchestra.run.smithi120.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:46:00.196 INFO:teuthology.orchestra.run.smithi120.stdout: Running scriptlet: kernel-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:46:00.196 INFO:teuthology.orchestra.run.smithi120.stdout: Verifying : kernel-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:46:00.196 INFO:teuthology.orchestra.run.smithi120.stdout: Verifying : kernel-core-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:46:00.648 INFO:teuthology.orchestra.run.smithi120.stdout: Verifying : kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:46:00.649 INFO:teuthology.orchestra.run.smithi120.stdout: 2024-04-10T22:46:00.649 INFO:teuthology.orchestra.run.smithi120.stdout:Installed: 2024-04-10T22:46:00.649 INFO:teuthology.orchestra.run.smithi120.stdout: kernel-4.18.0-552.el8.x86_64 kernel-core-4.18.0-552.el8.x86_64 2024-04-10T22:46:00.649 INFO:teuthology.orchestra.run.smithi120.stdout: kernel-modules-4.18.0-552.el8.x86_64 2024-04-10T22:46:00.649 INFO:teuthology.orchestra.run.smithi120.stdout: 2024-04-10T22:46:00.649 INFO:teuthology.orchestra.run.smithi120.stdout:Complete! 2024-04-10T22:46:00.832 DEBUG:teuthology.orchestra.run.smithi120:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:46:01.053 INFO:teuthology.orchestra.run.smithi120.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-10T22:46:01.054 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-10T22:46:01.054 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-552.el8.x86_64 2024-04-10T22:46:01.054 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-10T22:46:01.054 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-10T22:46:01.054 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-04-10T22:46:01.054 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi120.front.sepia.ceph.com, path=None, version=distro) 2024-04-10T22:46:01.054 DEBUG:teuthology.orchestra.run.smithi120:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:46:01.265 INFO:teuthology.orchestra.run.smithi120.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-10T22:46:01.266 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-10T22:46:01.266 DEBUG:teuthology.orchestra.run.smithi120:> sudo rpm -qi grub2-tools 2024-04-10T22:46:01.299 INFO:teuthology.orchestra.run.smithi120.stdout:Name : grub2-tools 2024-04-10T22:46:01.299 INFO:teuthology.orchestra.run.smithi120.stdout:Epoch : 1 2024-04-10T22:46:01.300 INFO:teuthology.orchestra.run.smithi120.stdout:Version : 2.02 2024-04-10T22:46:01.300 INFO:teuthology.orchestra.run.smithi120.stdout:Release : 129.el8 2024-04-10T22:46:01.300 INFO:teuthology.orchestra.run.smithi120.stdout:Architecture: x86_64 2024-04-10T22:46:01.300 INFO:teuthology.orchestra.run.smithi120.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2024-04-10T22:46:01.300 INFO:teuthology.orchestra.run.smithi120.stdout:Group : System Environment/Base 2024-04-10T22:46:01.300 INFO:teuthology.orchestra.run.smithi120.stdout:Size : 9265586 2024-04-10T22:46:01.300 INFO:teuthology.orchestra.run.smithi120.stdout:License : GPLv3+ 2024-04-10T22:46:01.300 INFO:teuthology.orchestra.run.smithi120.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2024-04-10T22:46:01.300 INFO:teuthology.orchestra.run.smithi120.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2024-04-10T22:46:01.300 INFO:teuthology.orchestra.run.smithi120.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2024-04-10T22:46:01.300 INFO:teuthology.orchestra.run.smithi120.stdout:Build Host : kbuilder.bsys.centos.org 2024-04-10T22:46:01.300 INFO:teuthology.orchestra.run.smithi120.stdout:Relocations : (not relocatable) 2024-04-10T22:46:01.300 INFO:teuthology.orchestra.run.smithi120.stdout:Packager : CentOS BuildSystem 2024-04-10T22:46:01.300 INFO:teuthology.orchestra.run.smithi120.stdout:Vendor : CentOS 2024-04-10T22:46:01.301 INFO:teuthology.orchestra.run.smithi120.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-10T22:46:01.301 INFO:teuthology.orchestra.run.smithi120.stdout:Summary : Support tools for GRUB. 2024-04-10T22:46:01.301 INFO:teuthology.orchestra.run.smithi120.stdout:Description : 2024-04-10T22:46:01.301 INFO:teuthology.orchestra.run.smithi120.stdout: 2024-04-10T22:46:01.301 INFO:teuthology.orchestra.run.smithi120.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-10T22:46:01.301 INFO:teuthology.orchestra.run.smithi120.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-10T22:46:01.301 INFO:teuthology.orchestra.run.smithi120.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-10T22:46:01.301 INFO:teuthology.orchestra.run.smithi120.stdout:hardware devices. 2024-04-10T22:46:01.301 INFO:teuthology.orchestra.run.smithi120.stdout: 2024-04-10T22:46:01.301 INFO:teuthology.orchestra.run.smithi120.stdout:This subpackage provides tools for support of all platforms. 2024-04-10T22:46:01.302 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-10T22:46:01.302 INFO:teuthology.task.kernel:Updating grub on smithi120 to boot 4.18.0-552.el8.x86_64 2024-04-10T22:46:01.302 DEBUG:teuthology.orchestra.run.smithi120:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-10T22:46:02.861 INFO:teuthology.orchestra.run.smithi120.stderr:Generating grub configuration file ... 2024-04-10T22:46:03.737 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:46:03.761 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:46:04.141 INFO:teuthology.orchestra.run.smithi151.stdout: Running scriptlet: kernel-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:46:04.142 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : kernel-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:46:04.142 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : kernel-core-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:46:04.648 INFO:teuthology.orchestra.run.smithi120.stderr:done 2024-04-10T22:46:04.649 DEBUG:teuthology.orchestra.run.smithi120:> mktemp 2024-04-10T22:46:04.664 INFO:teuthology.orchestra.run.smithi151.stdout: Verifying : kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:46:04.665 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-10T22:46:04.665 INFO:teuthology.orchestra.run.smithi151.stdout:Installed: 2024-04-10T22:46:04.665 INFO:teuthology.orchestra.run.smithi151.stdout: kernel-4.18.0-552.el8.x86_64 kernel-core-4.18.0-552.el8.x86_64 2024-04-10T22:46:04.665 INFO:teuthology.orchestra.run.smithi151.stdout: kernel-modules-4.18.0-552.el8.x86_64 2024-04-10T22:46:04.665 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-10T22:46:04.665 INFO:teuthology.orchestra.run.smithi151.stdout:Complete! 2024-04-10T22:46:04.690 INFO:teuthology.orchestra.run.smithi120.stdout:/tmp/tmp.Pb33sW59Sq 2024-04-10T22:46:04.690 DEBUG:teuthology.orchestra.run.smithi120:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.Pb33sW59Sq 2024-04-10T22:46:04.763 DEBUG:teuthology.orchestra.run.smithi120:> sudo chmod 0666 /tmp/tmp.Pb33sW59Sq 2024-04-10T22:46:04.790 DEBUG:teuthology.orchestra.run.smithi151:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:46:04.907 DEBUG:teuthology.orchestra.remote:smithi120:/tmp/tmp.Pb33sW59Sq is 6KB 2024-04-10T22:46:04.955 DEBUG:teuthology.orchestra.run.smithi120:> rm -fr /tmp/tmp.Pb33sW59Sq 2024-04-10T22:46:04.972 DEBUG:teuthology.orchestra.run.smithi120:> sudo /bin/ls /boot/loader/entries || true 2024-04-10T22:46:05.010 INFO:teuthology.orchestra.run.smithi151.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-10T22:46:05.010 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-10T22:46:05.011 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-552.el8.x86_64 2024-04-10T22:46:05.011 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-10T22:46:05.011 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-10T22:46:05.011 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-04-10T22:46:05.011 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi151.front.sepia.ceph.com, path=None, version=distro) 2024-04-10T22:46:05.011 DEBUG:teuthology.orchestra.run.smithi151:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:46:05.043 INFO:teuthology.orchestra.run.smithi120.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2024-04-10T22:46:05.044 INFO:teuthology.orchestra.run.smithi120.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2024-04-10T22:46:05.044 INFO:teuthology.orchestra.run.smithi120.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2024-04-10T22:46:05.044 INFO:teuthology.orchestra.run.smithi120.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64.conf 2024-04-10T22:46:05.045 DEBUG:teuthology.orchestra.run.smithi120:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64 2024-04-10T22:46:05.223 INFO:teuthology.orchestra.run.smithi151.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-10T22:46:05.224 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-10T22:46:05.224 DEBUG:teuthology.orchestra.run.smithi151:> sudo rpm -qi grub2-tools 2024-04-10T22:46:05.238 DEBUG:teuthology.orchestra.run.smithi120:> sudo shutdown -r now 2024-04-10T22:46:05.257 INFO:teuthology.orchestra.run.smithi151.stdout:Name : grub2-tools 2024-04-10T22:46:05.257 INFO:teuthology.orchestra.run.smithi151.stdout:Epoch : 1 2024-04-10T22:46:05.257 INFO:teuthology.orchestra.run.smithi151.stdout:Version : 2.02 2024-04-10T22:46:05.257 INFO:teuthology.orchestra.run.smithi151.stdout:Release : 129.el8 2024-04-10T22:46:05.257 INFO:teuthology.orchestra.run.smithi151.stdout:Architecture: x86_64 2024-04-10T22:46:05.257 INFO:teuthology.orchestra.run.smithi151.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2024-04-10T22:46:05.257 INFO:teuthology.orchestra.run.smithi151.stdout:Group : System Environment/Base 2024-04-10T22:46:05.257 INFO:teuthology.orchestra.run.smithi151.stdout:Size : 9265586 2024-04-10T22:46:05.257 INFO:teuthology.orchestra.run.smithi151.stdout:License : GPLv3+ 2024-04-10T22:46:05.257 INFO:teuthology.orchestra.run.smithi151.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2024-04-10T22:46:05.258 INFO:teuthology.orchestra.run.smithi151.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2024-04-10T22:46:05.258 INFO:teuthology.orchestra.run.smithi151.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2024-04-10T22:46:05.258 INFO:teuthology.orchestra.run.smithi151.stdout:Build Host : kbuilder.bsys.centos.org 2024-04-10T22:46:05.258 INFO:teuthology.orchestra.run.smithi151.stdout:Relocations : (not relocatable) 2024-04-10T22:46:05.258 INFO:teuthology.orchestra.run.smithi151.stdout:Packager : CentOS BuildSystem 2024-04-10T22:46:05.258 INFO:teuthology.orchestra.run.smithi151.stdout:Vendor : CentOS 2024-04-10T22:46:05.258 INFO:teuthology.orchestra.run.smithi151.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-10T22:46:05.258 INFO:teuthology.orchestra.run.smithi151.stdout:Summary : Support tools for GRUB. 2024-04-10T22:46:05.258 INFO:teuthology.orchestra.run.smithi151.stdout:Description : 2024-04-10T22:46:05.258 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-10T22:46:05.258 INFO:teuthology.orchestra.run.smithi151.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-10T22:46:05.258 INFO:teuthology.orchestra.run.smithi151.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-10T22:46:05.258 INFO:teuthology.orchestra.run.smithi151.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-10T22:46:05.258 INFO:teuthology.orchestra.run.smithi151.stdout:hardware devices. 2024-04-10T22:46:05.259 INFO:teuthology.orchestra.run.smithi151.stdout: 2024-04-10T22:46:05.259 INFO:teuthology.orchestra.run.smithi151.stdout:This subpackage provides tools for support of all platforms. 2024-04-10T22:46:05.259 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-10T22:46:05.260 INFO:teuthology.task.kernel:Updating grub on smithi151 to boot 4.18.0-552.el8.x86_64 2024-04-10T22:46:05.260 DEBUG:teuthology.orchestra.run.smithi151:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-10T22:46:06.897 INFO:teuthology.orchestra.run.smithi151.stderr:Generating grub configuration file ... 2024-04-10T22:46:09.169 INFO:teuthology.orchestra.run.smithi151.stderr:done 2024-04-10T22:46:09.171 DEBUG:teuthology.orchestra.run.smithi151:> mktemp 2024-04-10T22:46:09.212 INFO:teuthology.orchestra.run.smithi151.stdout:/tmp/tmp.wWARZtaONn 2024-04-10T22:46:09.212 DEBUG:teuthology.orchestra.run.smithi151:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.wWARZtaONn 2024-04-10T22:46:09.303 DEBUG:teuthology.orchestra.run.smithi151:> sudo chmod 0666 /tmp/tmp.wWARZtaONn 2024-04-10T22:46:09.400 DEBUG:teuthology.orchestra.remote:smithi151:/tmp/tmp.wWARZtaONn is 6KB 2024-04-10T22:46:09.448 DEBUG:teuthology.orchestra.run.smithi151:> rm -fr /tmp/tmp.wWARZtaONn 2024-04-10T22:46:09.466 DEBUG:teuthology.orchestra.run.smithi151:> sudo /bin/ls /boot/loader/entries || true 2024-04-10T22:46:09.551 INFO:teuthology.orchestra.run.smithi151.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2024-04-10T22:46:09.551 INFO:teuthology.orchestra.run.smithi151.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2024-04-10T22:46:09.551 INFO:teuthology.orchestra.run.smithi151.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2024-04-10T22:46:09.551 INFO:teuthology.orchestra.run.smithi151.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64.conf 2024-04-10T22:46:09.552 DEBUG:teuthology.orchestra.run.smithi151:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64 2024-04-10T22:46:09.671 DEBUG:teuthology.orchestra.run.smithi151:> sudo shutdown -r now 2024-04-10T22:46:35.266 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-04-10T22:46:35.267 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi120.front.sepia.ceph.com' 2024-04-10T22:46:35.268 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi120.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:46:39.678 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-04-10T22:46:39.679 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi151.front.sepia.ceph.com' 2024-04-10T22:46:39.679 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi151.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:47:09.855 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.120 2024-04-10T22:47:14.206 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.151 2024-04-10T22:47:18.858 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi120.front.sepia.ceph.com' 2024-04-10T22:47:18.859 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi120.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:47:23.212 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi151.front.sepia.ceph.com' 2024-04-10T22:47:23.212 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi151.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:47:37.246 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.120 2024-04-10T22:47:49.254 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi120.front.sepia.ceph.com' 2024-04-10T22:47:49.256 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi120.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:47:49.702 DEBUG:teuthology.orchestra.run.smithi120:> true 2024-04-10T22:47:49.975 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi120.front.sepia.ceph.com' 2024-04-10T22:47:49.975 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "4.18.0-552.el8.x86_64"... 2024-04-10T22:47:49.976 DEBUG:teuthology.orchestra.run.smithi120:> uname -r 2024-04-10T22:47:49.992 INFO:teuthology.orchestra.run.smithi120.stdout:4.18.0-552.el8.x86_64 2024-04-10T22:47:49.992 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-552.el8.x86_64 vs 4.18.0-552.el8.x86_64 2024-04-10T22:47:49.992 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-10T22:47:49.992 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-10T22:47:50.993 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-04-10T22:47:50.993 DEBUG:teuthology.orchestra.run.smithi120:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-10T22:47:51.127 INFO:teuthology.orchestra.run.smithi120.stdout:ttyS1 2024-04-10T22:47:51.138 DEBUG:teuthology.parallel:result is None 2024-04-10T22:48:23.215 DEBUG:teuthology.orchestra.remote:timed out 2024-04-10T22:48:35.216 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi151.front.sepia.ceph.com' 2024-04-10T22:48:35.217 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi151.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:48:35.604 DEBUG:teuthology.orchestra.run.smithi151:> true 2024-04-10T22:48:35.897 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi151.front.sepia.ceph.com' 2024-04-10T22:48:35.897 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "4.18.0-552.el8.x86_64"... 2024-04-10T22:48:35.897 DEBUG:teuthology.orchestra.run.smithi151:> uname -r 2024-04-10T22:48:35.917 INFO:teuthology.orchestra.run.smithi151.stdout:4.18.0-552.el8.x86_64 2024-04-10T22:48:35.917 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-552.el8.x86_64 vs 4.18.0-552.el8.x86_64 2024-04-10T22:48:35.917 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-10T22:48:35.917 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-10T22:48:36.918 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-04-10T22:48:36.919 DEBUG:teuthology.orchestra.run.smithi151:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-10T22:48:36.985 INFO:teuthology.orchestra.run.smithi151.stdout:ttyS1 2024-04-10T22:48:37.000 DEBUG:teuthology.parallel:result is None 2024-04-10T22:48:37.000 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-10T22:48:37.007 INFO:teuthology.task.internal:Creating test directory... 2024-04-10T22:48:37.007 DEBUG:teuthology.orchestra.run.smithi120:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-10T22:48:37.010 DEBUG:teuthology.orchestra.run.smithi151:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-10T22:48:37.031 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-10T22:48:37.038 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-10T22:48:37.044 INFO:teuthology.task.internal:Creating archive directory... 2024-04-10T22:48:37.044 DEBUG:teuthology.orchestra.run.smithi120:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-10T22:48:37.073 DEBUG:teuthology.orchestra.run.smithi151:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-10T22:48:37.124 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-10T22:48:37.130 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-10T22:48:37.131 DEBUG:teuthology.orchestra.run.smithi120:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-10T22:48:37.133 DEBUG:teuthology.orchestra.run.smithi151:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-10T22:48:37.168 INFO:teuthology.orchestra.run.smithi120.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:48:37.179 INFO:teuthology.orchestra.run.smithi120.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:48:37.191 INFO:teuthology.orchestra.run.smithi151.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:48:37.202 INFO:teuthology.orchestra.run.smithi151.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:48:37.203 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-10T22:48:37.243 INFO:teuthology.task.internal:Configuring sudo... 2024-04-10T22:48:37.243 DEBUG:teuthology.orchestra.run.smithi120:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-10T22:48:37.246 DEBUG:teuthology.orchestra.run.smithi151:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-10T22:48:37.275 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-10T22:48:37.303 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-10T22:48:37.303 DEBUG:teuthology.orchestra.run.smithi120:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-10T22:48:37.317 DEBUG:teuthology.orchestra.run.smithi151:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-10T22:48:37.336 DEBUG:teuthology.orchestra.run.smithi120:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:48:37.422 DEBUG:teuthology.orchestra.run.smithi120:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:48:37.471 DEBUG:teuthology.orchestra.run.smithi120:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:48:37.555 DEBUG:teuthology.orchestra.run.smithi120:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:48:37.598 DEBUG:teuthology.orchestra.run.smithi120:> set -ex 2024-04-10T22:48:37.598 DEBUG:teuthology.orchestra.run.smithi120:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-10T22:48:37.664 DEBUG:teuthology.orchestra.run.smithi151:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:48:37.710 DEBUG:teuthology.orchestra.run.smithi151:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:48:37.761 DEBUG:teuthology.orchestra.run.smithi151:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:48:37.848 DEBUG:teuthology.orchestra.run.smithi151:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:48:37.890 DEBUG:teuthology.orchestra.run.smithi151:> set -ex 2024-04-10T22:48:37.890 DEBUG:teuthology.orchestra.run.smithi151:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-10T22:48:37.957 DEBUG:teuthology.orchestra.run.smithi120:> sudo service rsyslog restart 2024-04-10T22:48:37.960 DEBUG:teuthology.orchestra.run.smithi151:> sudo service rsyslog restart 2024-04-10T22:48:38.035 INFO:teuthology.orchestra.run.smithi120.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:48:38.064 INFO:teuthology.orchestra.run.smithi151.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:48:38.450 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-10T22:48:38.461 INFO:teuthology.task.internal:Starting timer... 2024-04-10T22:48:38.461 INFO:teuthology.run_tasks:Running task pcp... 2024-04-10T22:48:38.529 INFO:teuthology.run_tasks:Running task selinux... 2024-04-10T22:48:38.537 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-04-10T22:48:38.537 DEBUG:teuthology.orchestra.run.smithi120:> sudo service auditd rotate 2024-04-10T22:48:38.615 INFO:teuthology.orchestra.run.smithi120.stdout:Rotating logs: 2024-04-10T22:48:38.617 DEBUG:teuthology.orchestra.run.smithi151:> sudo service auditd rotate 2024-04-10T22:48:38.690 INFO:teuthology.orchestra.run.smithi151.stdout:Rotating logs: 2024-04-10T22:48:38.691 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-10T22:48:38.691 DEBUG:teuthology.orchestra.run.smithi120:> /usr/sbin/getenforce 2024-04-10T22:48:38.749 INFO:teuthology.orchestra.run.smithi120.stdout:Permissive 2024-04-10T22:48:38.749 DEBUG:teuthology.orchestra.run.smithi151:> /usr/sbin/getenforce 2024-04-10T22:48:38.801 INFO:teuthology.orchestra.run.smithi151.stdout:Permissive 2024-04-10T22:48:38.801 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi120.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi151.front.sepia.ceph.com': 'permissive'} 2024-04-10T22:48:38.801 DEBUG:teuthology.orchestra.run.smithi120:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:48:38.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:48:38.830 DEBUG:teuthology.orchestra.run.smithi151:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:48:38.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:48:38.858 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-10T22:48:38.858 DEBUG:teuthology.orchestra.run.smithi120:> sudo /usr/sbin/setenforce permissive 2024-04-10T22:48:38.897 DEBUG:teuthology.orchestra.run.smithi151:> sudo /usr/sbin/setenforce permissive 2024-04-10T22:48:38.924 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-10T22:48:38.934 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-10T22:48:38.951 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-10T22:48:38.952 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi120.front.sepia.ceph.com,smithi151.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-10T23:01:24.744 INFO:teuthology.task.ansible:Archiving ansible failure log at: /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650183/ansible_failures.yaml 2024-04-10T23:01:24.746 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/__init__.py", line 123, in __enter__ self.begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 508, in begin super(CephLab, self).begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 342, in begin self.execute_playbook() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 374, in execute_playbook self._handle_failure(command, status) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 402, in _handle_failure raise AnsibleFailedError(failures) teuthology.exceptions.AnsibleFailedError: [',', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at'] 2024-04-10T23:01:24.957 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=d2f31874c67e4cc982857e196f136bf0 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/__init__.py", line 123, in __enter__ self.begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 508, in begin super(CephLab, self).begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 342, in begin self.execute_playbook() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 374, in execute_playbook self._handle_failure(command, status) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 402, in _handle_failure raise AnsibleFailedError(failures) teuthology.exceptions.AnsibleFailedError: [',', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at'] 2024-04-10T23:01:24.959 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-10T23:01:24.967 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-10T23:01:24.967 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-10T23:01:24.975 DEBUG:teuthology.orchestra.run.smithi120:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-10T23:01:25.056 DEBUG:teuthology.orchestra.run.smithi151:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-10T23:01:25.123 DEBUG:teuthology.orchestra.run.smithi120:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T23:01:25.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T23:01:25.153 DEBUG:teuthology.orchestra.run.smithi151:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T23:01:25.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T23:01:25.184 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-10T23:01:25.193 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-10T23:01:25.200 INFO:teuthology.task.internal:Duration was 766.738645 seconds 2024-04-10T23:01:25.200 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-10T23:01:25.208 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-10T23:01:25.208 DEBUG:teuthology.orchestra.run.smithi120:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-10T23:01:25.210 DEBUG:teuthology.orchestra.run.smithi151:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-10T23:01:25.255 INFO:teuthology.orchestra.run.smithi120.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T23:01:25.272 INFO:teuthology.orchestra.run.smithi151.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T23:01:25.454 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-10T23:01:25.455 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi120.front.sepia.ceph.com 2024-04-10T23:01:25.455 DEBUG:teuthology.orchestra.run.smithi120:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-10T23:01:25.477 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi151.front.sepia.ceph.com 2024-04-10T23:01:25.477 DEBUG:teuthology.orchestra.run.smithi151:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-10T23:01:25.500 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-10T23:01:25.500 DEBUG:teuthology.orchestra.run.smithi120:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-10T23:01:25.520 DEBUG:teuthology.orchestra.run.smithi151:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-10T23:01:25.580 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-10T23:01:25.580 DEBUG:teuthology.orchestra.run.smithi120:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-10T23:01:25.600 DEBUG:teuthology.orchestra.run.smithi151:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-10T23:01:25.681 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-10T23:01:25.691 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-10T23:01:25.691 DEBUG:teuthology.orchestra.run.smithi120:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-10T23:01:25.718 DEBUG:teuthology.orchestra.run.smithi151:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-10T23:01:25.745 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-10T23:01:25.753 DEBUG:teuthology.orchestra.run.smithi120:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-10T23:01:25.760 DEBUG:teuthology.orchestra.run.smithi151:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-10T23:01:25.784 INFO:teuthology.orchestra.run.smithi120.stdout:kernel.core_pattern = core 2024-04-10T23:01:25.811 INFO:teuthology.orchestra.run.smithi151.stdout:kernel.core_pattern = core 2024-04-10T23:01:25.842 DEBUG:teuthology.orchestra.run.smithi120:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-10T23:01:25.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T23:01:25.859 DEBUG:teuthology.orchestra.run.smithi151:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-10T23:01:25.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T23:01:25.900 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-10T23:01:25.908 INFO:teuthology.task.internal:Transferring archived files... 2024-04-10T23:01:25.910 DEBUG:teuthology.misc:Transferring archived files from smithi120:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650183/remote/smithi120 2024-04-10T23:01:25.910 DEBUG:teuthology.orchestra.run.smithi120:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-10T23:01:25.967 DEBUG:teuthology.misc:Transferring archived files from smithi151:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650183/remote/smithi151 2024-04-10T23:01:25.968 DEBUG:teuthology.orchestra.run.smithi151:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-10T23:01:26.022 INFO:teuthology.task.internal:Removing archive directory... 2024-04-10T23:01:26.022 DEBUG:teuthology.orchestra.run.smithi120:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-10T23:01:26.024 DEBUG:teuthology.orchestra.run.smithi151:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-10T23:01:26.074 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-10T23:01:26.084 INFO:teuthology.task.internal:Not uploading archives. 2024-04-10T23:01:26.085 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-10T23:01:26.093 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-10T23:01:26.093 DEBUG:teuthology.orchestra.run.smithi120:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-10T23:01:26.095 DEBUG:teuthology.orchestra.run.smithi151:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-10T23:01:26.110 INFO:teuthology.orchestra.run.smithi120.stdout: 393224 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 10 23:01 /home/ubuntu/cephtest 2024-04-10T23:01:26.131 INFO:teuthology.orchestra.run.smithi151.stdout: 393224 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 10 23:01 /home/ubuntu/cephtest 2024-04-10T23:01:26.133 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-10T23:01:26.142 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-10T23:01:26.159 INFO:teuthology.nuke:Checking targets against current locks 2024-04-10T23:01:26.191 DEBUG:teuthology.nuke:shortname: smithi120 2024-04-10T23:01:26.192 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T23:01:26.234 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi120.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650183', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:16.683421', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDORNFmvtZrQHmn/4/z3NftYfW4emO0WurdXs4mBiQhR6laI7LK10k6Z3ywA4IF7ZvsU/EnwWLswiOmZXAty/d0='} 2024-04-10T23:01:26.248 DEBUG:teuthology.nuke:shortname: smithi151 2024-04-10T23:01:26.249 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T23:01:26.291 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi151.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650183', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:16.687307', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCQEu8BXZEJ6hwD8DeLEyN6GMDvjUgnEGx3k5IRicfbZEbNaXMzn2ELP+N9c04VI4mcDHA7O2+7+JlP/QZFyDAE='} 2024-04-10T23:01:26.341 INFO:teuthology.orchestra.console.smithi120:Power off 2024-04-10T23:01:26.341 DEBUG:teuthology.orchestra.console.smithi120:pexpect command: ipmitool -H smithi120.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-10T23:01:26.369 INFO:teuthology.orchestra.console.smithi151:Power off 2024-04-10T23:01:26.370 DEBUG:teuthology.orchestra.console.smithi151:pexpect command: ipmitool -H smithi151.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-10T23:01:26.392 DEBUG:teuthology.orchestra.console.smithi120:power off output: Chassis Power Control: Down/Off 2024-04-10T23:01:26.393 DEBUG:teuthology.orchestra.console.smithi120:pexpect command: ipmitool -H smithi120.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:01:26.420 DEBUG:teuthology.orchestra.console.smithi151:power off output: Chassis Power Control: Down/Off 2024-04-10T23:01:26.421 DEBUG:teuthology.orchestra.console.smithi151:pexpect command: ipmitool -H smithi151.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:01:26.443 DEBUG:teuthology.orchestra.console.smithi120:check power output: Chassis Power is on 2024-04-10T23:01:26.470 DEBUG:teuthology.orchestra.console.smithi151:check power output: Chassis Power is on 2024-04-10T23:01:30.444 DEBUG:teuthology.orchestra.console.smithi120:pexpect command: ipmitool -H smithi120.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:01:30.471 DEBUG:teuthology.orchestra.console.smithi151:pexpect command: ipmitool -H smithi151.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:01:30.556 DEBUG:teuthology.orchestra.console.smithi120:check power output: Chassis Power is on 2024-04-10T23:01:30.582 DEBUG:teuthology.orchestra.console.smithi151:check power output: Chassis Power is on 2024-04-10T23:01:34.559 DEBUG:teuthology.orchestra.console.smithi120:pexpect command: ipmitool -H smithi120.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:01:34.582 DEBUG:teuthology.orchestra.console.smithi151:pexpect command: ipmitool -H smithi151.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:01:34.670 DEBUG:teuthology.orchestra.console.smithi120:check power output: Chassis Power is on 2024-04-10T23:01:34.691 DEBUG:teuthology.orchestra.console.smithi151:check power output: Chassis Power is off 2024-04-10T23:01:34.791 INFO:teuthology.orchestra.console.smithi151:Power off completed 2024-04-10T23:01:38.674 DEBUG:teuthology.orchestra.console.smithi120:pexpect command: ipmitool -H smithi120.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:01:38.785 DEBUG:teuthology.orchestra.console.smithi120:check power output: Chassis Power is off 2024-04-10T23:01:38.886 INFO:teuthology.orchestra.console.smithi120:Power off completed 2024-04-10T23:01:38.992 INFO:teuthology.run:Summary data: description: upgrade:pacific-x/stress-split/{0-distro/centos_8.stream_container_tools 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} duration: 766.7386448383331 failure_reason: '['','', ''Bugzilla. In case of problems, contact the owner of this repository.'', ''Enabling a Copr repository. Please note that this repository is not part'', ''Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable'', ''Please do not file bug reports about these packages in Fedora'', ''The Fedora Project does not exercise any power over the contents of'', ''and packages are not held to any quality or security level.'', ''of the main distribution, and quality may vary.'', ''this repository beyond the rules outlined in the Copr FAQ at'']' owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=d2f31874c67e4cc982857e196f136bf0 status: dead success: false 2024-04-10T23:01:38.992 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-10T23:01:39.087 INFO:teuthology.run:DEAD