2024-04-10T22:38:31.635 INFO:root:teuthology version: 0.0.1.dev267+g6c63784 2024-04-10T22:38:31.635 DEBUG:teuthology.run:Teuthology command: teuthology --archive /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650189 --description upgrade:pacific-x/stress-split/{0-distro/centos_8.stream_container_tools_crun 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} --owner scheduled_teuthology@teuthology --verbose --name teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi -- /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650189/orig.config.yaml 2024-04-10T22:38:31.692 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-10T22:38:31.785 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650189 branch: reef description: upgrade:pacific-x/stress-split/{0-distro/centos_8.stream_container_tools_crun 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} email: ceph-qa@ceph.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: pacific clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' first_in_suite: false job_id: '7650189' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'librbd C and C++ api tests ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run basic import/export cli tests for rbd ' name: teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 8.stream overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: reef sha1: 4f70a9a90881107996c8b7a3e89ba8b32624713c owner: scheduled_teuthology@teuthology priority: 830 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' seed: 4604 sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: pacific clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 0/1 suite: upgrade:pacific-x suite_branch: reef suite_path: /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 4f70a9a90881107996c8b7a3e89ba8b32624713c targets: smithi032.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFMfiV2F3C5RwI85TIdaj6uPGhdnQMnccTu7MUJZMX1rG3qn976HpbwkSLmeauSIKOXQRWx8RTUQRbuXJTaId4U= smithi194.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGh/6nZGaFxmOMax7pKbRq6YsdK+3vj8be0kmr3iGQEI16T9Isv6/VLGkIumJp4FfwCTO6hQaaIOW0iUZNCeIrc= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - sudo sed -i 's/runtime = "runc"/#runtime = "runc"/g' /usr/share/containers/containers.conf - sudo sed -i 's/#runtime = "crun"/runtime = "crun"/g' /usr/share/containers/containers.conf - install: branch: pacific exclude_packages: - ceph-volume - cephadm: cephadm_branch: pacific cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:pacific - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 6c637841c215537a4502385240412f1966e0faab timestamp: 2024-04-10_01:08:08 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-04-10T22:38:31.785 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa; will attempt to use it 2024-04-10T22:38:31.786 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa/tasks 2024-04-10T22:38:31.786 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-10T22:38:31.788 INFO:teuthology.task.internal:Checking packages... 2024-04-10T22:38:31.809 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'd540ebaca6b131a1dd560e7f69e024b133bbaa42' 2024-04-10T22:38:31.809 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-10T22:38:31.810 INFO:teuthology.packaging:ref: None 2024-04-10T22:38:31.810 INFO:teuthology.packaging:tag: None 2024-04-10T22:38:31.810 INFO:teuthology.packaging:branch: reef 2024-04-10T22:38:31.810 INFO:teuthology.packaging:sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 2024-04-10T22:38:31.810 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef 2024-04-10T22:38:32.219 INFO:teuthology.task.internal:Found packages for ceph version 18.2.2-1189.gc96c8d47 2024-04-10T22:38:32.221 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-10T22:38:32.227 INFO:teuthology.task.internal:no buildpackages task found 2024-04-10T22:38:32.228 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-10T22:38:32.236 INFO:teuthology.task.internal:Saving configuration 2024-04-10T22:38:32.258 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-10T22:38:32.264 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T22:38:32.287 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi032.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650189', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:19.221626', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFMfiV2F3C5RwI85TIdaj6uPGhdnQMnccTu7MUJZMX1rG3qn976HpbwkSLmeauSIKOXQRWx8RTUQRbuXJTaId4U='} 2024-04-10T22:38:32.310 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi194.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650189', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:19.223105', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGh/6nZGaFxmOMax7pKbRq6YsdK+3vj8be0kmr3iGQEI16T9Isv6/VLGkIumJp4FfwCTO6hQaaIOW0iUZNCeIrc='} 2024-04-10T22:38:32.310 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-10T22:38:32.316 INFO:teuthology.task.internal:roles: ubuntu@smithi032.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-04-10T22:38:32.316 INFO:teuthology.task.internal:roles: ubuntu@smithi194.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-04-10T22:38:32.316 INFO:teuthology.run_tasks:Running task console_log... 2024-04-10T22:38:32.380 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fe9329fc0d0>, signals=[15]) 2024-04-10T22:38:32.380 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-10T22:38:32.387 INFO:teuthology.task.internal:Opening connections... 2024-04-10T22:38:32.387 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi032.front.sepia.ceph.com 2024-04-10T22:38:32.389 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi032.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:38:32.465 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi194.front.sepia.ceph.com 2024-04-10T22:38:32.466 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi194.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:38:32.537 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-10T22:38:32.544 DEBUG:teuthology.orchestra.run.smithi032:> uname -m 2024-04-10T22:38:32.567 INFO:teuthology.orchestra.run.smithi032.stdout:x86_64 2024-04-10T22:38:32.567 DEBUG:teuthology.orchestra.run.smithi032:> cat /etc/os-release 2024-04-10T22:38:32.631 INFO:teuthology.orchestra.run.smithi032.stdout:NAME="CentOS Stream" 2024-04-10T22:38:32.631 INFO:teuthology.orchestra.run.smithi032.stdout:VERSION="8" 2024-04-10T22:38:32.631 INFO:teuthology.orchestra.run.smithi032.stdout:ID="centos" 2024-04-10T22:38:32.631 INFO:teuthology.orchestra.run.smithi032.stdout:ID_LIKE="rhel fedora" 2024-04-10T22:38:32.631 INFO:teuthology.orchestra.run.smithi032.stdout:VERSION_ID="8" 2024-04-10T22:38:32.631 INFO:teuthology.orchestra.run.smithi032.stdout:PLATFORM_ID="platform:el8" 2024-04-10T22:38:32.631 INFO:teuthology.orchestra.run.smithi032.stdout:PRETTY_NAME="CentOS Stream 8" 2024-04-10T22:38:32.631 INFO:teuthology.orchestra.run.smithi032.stdout:ANSI_COLOR="0;31" 2024-04-10T22:38:32.631 INFO:teuthology.orchestra.run.smithi032.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2024-04-10T22:38:32.631 INFO:teuthology.orchestra.run.smithi032.stdout:HOME_URL="https://centos.org/" 2024-04-10T22:38:32.632 INFO:teuthology.orchestra.run.smithi032.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-10T22:38:32.632 INFO:teuthology.orchestra.run.smithi032.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2024-04-10T22:38:32.632 INFO:teuthology.orchestra.run.smithi032.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-10T22:38:32.632 INFO:teuthology.lock.ops:Updating smithi032.front.sepia.ceph.com on lock server 2024-04-10T22:38:32.652 DEBUG:teuthology.orchestra.run.smithi194:> uname -m 2024-04-10T22:38:32.674 INFO:teuthology.orchestra.run.smithi194.stdout:x86_64 2024-04-10T22:38:32.674 DEBUG:teuthology.orchestra.run.smithi194:> cat /etc/os-release 2024-04-10T22:38:32.734 INFO:teuthology.orchestra.run.smithi194.stdout:NAME="CentOS Stream" 2024-04-10T22:38:32.734 INFO:teuthology.orchestra.run.smithi194.stdout:VERSION="8" 2024-04-10T22:38:32.734 INFO:teuthology.orchestra.run.smithi194.stdout:ID="centos" 2024-04-10T22:38:32.734 INFO:teuthology.orchestra.run.smithi194.stdout:ID_LIKE="rhel fedora" 2024-04-10T22:38:32.734 INFO:teuthology.orchestra.run.smithi194.stdout:VERSION_ID="8" 2024-04-10T22:38:32.734 INFO:teuthology.orchestra.run.smithi194.stdout:PLATFORM_ID="platform:el8" 2024-04-10T22:38:32.735 INFO:teuthology.orchestra.run.smithi194.stdout:PRETTY_NAME="CentOS Stream 8" 2024-04-10T22:38:32.735 INFO:teuthology.orchestra.run.smithi194.stdout:ANSI_COLOR="0;31" 2024-04-10T22:38:32.735 INFO:teuthology.orchestra.run.smithi194.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2024-04-10T22:38:32.735 INFO:teuthology.orchestra.run.smithi194.stdout:HOME_URL="https://centos.org/" 2024-04-10T22:38:32.735 INFO:teuthology.orchestra.run.smithi194.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-10T22:38:32.735 INFO:teuthology.orchestra.run.smithi194.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2024-04-10T22:38:32.735 INFO:teuthology.orchestra.run.smithi194.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-10T22:38:32.735 INFO:teuthology.lock.ops:Updating smithi194.front.sepia.ceph.com on lock server 2024-04-10T22:38:32.751 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-10T22:38:32.761 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-10T22:38:32.766 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-10T22:38:32.766 DEBUG:teuthology.orchestra.run.smithi032:> test '!' -e /home/ubuntu/cephtest 2024-04-10T22:38:32.769 DEBUG:teuthology.orchestra.run.smithi194:> test '!' -e /home/ubuntu/cephtest 2024-04-10T22:38:32.795 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-10T22:38:32.801 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-10T22:38:32.801 DEBUG:teuthology.orchestra.run.smithi032:> test -z $(ls -A /var/lib/ceph) 2024-04-10T22:38:32.830 DEBUG:teuthology.orchestra.run.smithi194:> test -z $(ls -A /var/lib/ceph) 2024-04-10T22:38:32.867 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-10T22:38:32.921 INFO:teuthology.run_tasks:Running task kernel... 2024-04-10T22:38:32.935 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-04-10T22:38:32.935 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-04-10T22:38:32.935 DEBUG:teuthology.orchestra.run.smithi032:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-10T22:38:32.936 DEBUG:teuthology.orchestra.run.smithi194:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-10T22:38:32.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:38:32.956 DEBUG:teuthology.orchestra.run.smithi194:> uname -r 2024-04-10T22:38:32.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:38:32.956 DEBUG:teuthology.orchestra.run.smithi032:> uname -r 2024-04-10T22:38:33.016 INFO:teuthology.orchestra.run.smithi194.stdout:4.18.0-489.el8.x86_64 2024-04-10T22:38:33.016 INFO:teuthology.task.kernel:Running kernel on smithi194: 4.18.0-489.el8.x86_64 2024-04-10T22:38:33.016 DEBUG:teuthology.orchestra.run.smithi194:> sudo yum install -y kernel 2024-04-10T22:38:33.018 INFO:teuthology.orchestra.run.smithi032.stdout:4.18.0-489.el8.x86_64 2024-04-10T22:38:33.018 INFO:teuthology.task.kernel:Running kernel on smithi032: 4.18.0-489.el8.x86_64 2024-04-10T22:38:33.019 DEBUG:teuthology.orchestra.run.smithi032:> sudo yum install -y kernel 2024-04-10T22:38:36.838 INFO:teuthology.orchestra.run.smithi194.stdout:CentOS Stream 8 - AppStream 21 MB/s | 28 MB 00:01 2024-04-10T22:40:45.992 INFO:teuthology.orchestra.run.smithi032.stdout:CentOS Stream 8 - AppStream 219 kB/s | 28 MB 02:10 2024-04-10T22:40:52.534 INFO:teuthology.orchestra.run.smithi032.stdout:CentOS Stream 8 - BaseOS 8.4 MB/s | 10 MB 00:01 2024-04-10T22:40:54.077 INFO:teuthology.orchestra.run.smithi194.stdout:CentOS Stream 8 - BaseOS 80 kB/s | 10 MB 02:12 2024-04-10T22:40:54.676 INFO:teuthology.orchestra.run.smithi032.stdout:CentOS Stream 8 - Extras 41 kB/s | 18 kB 00:00 2024-04-10T22:40:56.094 INFO:teuthology.orchestra.run.smithi194.stdout:CentOS Stream 8 - Extras 42 kB/s | 18 kB 00:00 2024-04-10T22:40:56.545 INFO:teuthology.orchestra.run.smithi194.stdout:CentOS Stream 8 - Extras common packages 22 kB/s | 7.7 kB 00:00 2024-04-10T22:41:36.322 INFO:teuthology.orchestra.run.smithi194.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:39 2024-04-10T22:41:36.323 INFO:teuthology.orchestra.run.smithi194.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:41:36.323 INFO:teuthology.orchestra.run.smithi194.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:41:36.323 INFO:teuthology.orchestra.run.smithi194.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:41:38.435 INFO:teuthology.orchestra.run.smithi194.stdout:Extra Packages for Enterprise Linux 7.7 MB/s | 16 MB 00:02 2024-04-10T22:41:42.182 INFO:teuthology.orchestra.run.smithi194.stdout:lab-extras 488 kB/s | 24 kB 00:00 2024-04-10T22:41:42.326 INFO:teuthology.orchestra.run.smithi194.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:41:45.154 INFO:teuthology.orchestra.run.smithi194.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2024-04-10T22:41:45.155 INFO:teuthology.orchestra.run.smithi194.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2024-04-10T22:41:45.258 INFO:teuthology.orchestra.run.smithi194.stdout:Dependencies resolved. 2024-04-10T22:41:45.259 INFO:teuthology.orchestra.run.smithi194.stdout:================================================================================ 2024-04-10T22:41:45.259 INFO:teuthology.orchestra.run.smithi194.stdout: Package Architecture Version Repository Size 2024-04-10T22:41:45.259 INFO:teuthology.orchestra.run.smithi194.stdout:================================================================================ 2024-04-10T22:41:45.259 INFO:teuthology.orchestra.run.smithi194.stdout:Installing: 2024-04-10T22:41:45.259 INFO:teuthology.orchestra.run.smithi194.stdout: kernel x86_64 4.18.0-552.el8 baseos 10 M 2024-04-10T22:41:45.260 INFO:teuthology.orchestra.run.smithi194.stdout:Installing dependencies: 2024-04-10T22:41:45.260 INFO:teuthology.orchestra.run.smithi194.stdout: kernel-core x86_64 4.18.0-552.el8 baseos 44 M 2024-04-10T22:41:45.260 INFO:teuthology.orchestra.run.smithi194.stdout: kernel-modules x86_64 4.18.0-552.el8 baseos 36 M 2024-04-10T22:41:45.260 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-04-10T22:41:45.260 INFO:teuthology.orchestra.run.smithi194.stdout:Transaction Summary 2024-04-10T22:41:45.260 INFO:teuthology.orchestra.run.smithi194.stdout:================================================================================ 2024-04-10T22:41:45.260 INFO:teuthology.orchestra.run.smithi194.stdout:Install 3 Packages 2024-04-10T22:41:45.260 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-04-10T22:41:45.260 INFO:teuthology.orchestra.run.smithi194.stdout:Total download size: 90 M 2024-04-10T22:41:45.260 INFO:teuthology.orchestra.run.smithi194.stdout:Installed size: 96 M 2024-04-10T22:41:45.260 INFO:teuthology.orchestra.run.smithi194.stdout:Downloading Packages: 2024-04-10T22:43:04.122 INFO:teuthology.orchestra.run.smithi032.stdout:CentOS Stream 8 - Extras common packages 61 B/s | 7.7 kB 02:09 2024-04-10T22:43:47.918 INFO:teuthology.orchestra.run.smithi032.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:43 2024-04-10T22:43:47.919 INFO:teuthology.orchestra.run.smithi032.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:43:47.919 INFO:teuthology.orchestra.run.smithi032.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:43:47.919 INFO:teuthology.orchestra.run.smithi032.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:44:01.522 INFO:teuthology.orchestra.run.smithi194.stdout:(1/3): kernel-4.18.0-552.el8.x86_64.rpm 2.6 MB/s | 10 MB 00:04 2024-04-10T22:44:12.933 INFO:teuthology.orchestra.run.smithi194.stdout:(2/3): kernel-modules-4.18.0-552.el8.x86_64.rpm 2.4 MB/s | 36 MB 00:15 2024-04-10T22:44:24.202 INFO:teuthology.orchestra.run.smithi194.stdout:(3/3): kernel-core-4.18.0-552.el8.x86_64.rpm 1.6 MB/s | 44 MB 00:26 2024-04-10T22:44:24.202 INFO:teuthology.orchestra.run.smithi194.stdout:-------------------------------------------------------------------------------- 2024-04-10T22:44:24.203 INFO:teuthology.orchestra.run.smithi194.stdout:Total 583 kB/s | 90 MB 02:38 2024-04-10T22:44:24.892 INFO:teuthology.orchestra.run.smithi194.stdout:Running transaction check 2024-04-10T22:44:25.322 INFO:teuthology.orchestra.run.smithi194.stdout:Transaction check succeeded. 2024-04-10T22:44:25.322 INFO:teuthology.orchestra.run.smithi194.stdout:Running transaction test 2024-04-10T22:44:26.589 INFO:teuthology.orchestra.run.smithi194.stdout:Transaction test succeeded. 2024-04-10T22:44:26.621 INFO:teuthology.orchestra.run.smithi194.stdout:Running transaction 2024-04-10T22:44:29.411 INFO:teuthology.orchestra.run.smithi194.stdout: Preparing : 1/1 2024-04-10T22:44:29.991 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:44:33.571 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:44:33.941 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:44:37.374 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:44:37.648 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : kernel-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:44:59.474 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:44:59.498 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:44:59.836 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: kernel-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:44:59.836 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : kernel-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:44:59.837 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : kernel-core-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:45:00.324 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:45:00.324 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-04-10T22:45:00.325 INFO:teuthology.orchestra.run.smithi194.stdout:Installed: 2024-04-10T22:45:00.325 INFO:teuthology.orchestra.run.smithi194.stdout: kernel-4.18.0-552.el8.x86_64 kernel-core-4.18.0-552.el8.x86_64 2024-04-10T22:45:00.325 INFO:teuthology.orchestra.run.smithi194.stdout: kernel-modules-4.18.0-552.el8.x86_64 2024-04-10T22:45:00.325 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-04-10T22:45:00.325 INFO:teuthology.orchestra.run.smithi194.stdout:Complete! 2024-04-10T22:45:00.473 DEBUG:teuthology.orchestra.run.smithi194:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:45:00.693 INFO:teuthology.orchestra.run.smithi194.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-10T22:45:00.694 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-10T22:45:00.694 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-552.el8.x86_64 2024-04-10T22:45:00.694 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-10T22:45:00.694 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-10T22:45:00.694 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-04-10T22:45:00.695 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi194.front.sepia.ceph.com, path=None, version=distro) 2024-04-10T22:45:00.695 DEBUG:teuthology.orchestra.run.smithi194:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:45:00.911 INFO:teuthology.orchestra.run.smithi194.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-10T22:45:00.911 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-10T22:45:00.911 DEBUG:teuthology.orchestra.run.smithi194:> sudo rpm -qi grub2-tools 2024-04-10T22:45:00.945 INFO:teuthology.orchestra.run.smithi194.stdout:Name : grub2-tools 2024-04-10T22:45:00.945 INFO:teuthology.orchestra.run.smithi194.stdout:Epoch : 1 2024-04-10T22:45:00.945 INFO:teuthology.orchestra.run.smithi194.stdout:Version : 2.02 2024-04-10T22:45:00.945 INFO:teuthology.orchestra.run.smithi194.stdout:Release : 129.el8 2024-04-10T22:45:00.945 INFO:teuthology.orchestra.run.smithi194.stdout:Architecture: x86_64 2024-04-10T22:45:00.945 INFO:teuthology.orchestra.run.smithi194.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2024-04-10T22:45:00.945 INFO:teuthology.orchestra.run.smithi194.stdout:Group : System Environment/Base 2024-04-10T22:45:00.946 INFO:teuthology.orchestra.run.smithi194.stdout:Size : 9265586 2024-04-10T22:45:00.946 INFO:teuthology.orchestra.run.smithi194.stdout:License : GPLv3+ 2024-04-10T22:45:00.946 INFO:teuthology.orchestra.run.smithi194.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2024-04-10T22:45:00.946 INFO:teuthology.orchestra.run.smithi194.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2024-04-10T22:45:00.946 INFO:teuthology.orchestra.run.smithi194.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2024-04-10T22:45:00.946 INFO:teuthology.orchestra.run.smithi194.stdout:Build Host : kbuilder.bsys.centos.org 2024-04-10T22:45:00.946 INFO:teuthology.orchestra.run.smithi194.stdout:Relocations : (not relocatable) 2024-04-10T22:45:00.946 INFO:teuthology.orchestra.run.smithi194.stdout:Packager : CentOS BuildSystem 2024-04-10T22:45:00.946 INFO:teuthology.orchestra.run.smithi194.stdout:Vendor : CentOS 2024-04-10T22:45:00.946 INFO:teuthology.orchestra.run.smithi194.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-10T22:45:00.947 INFO:teuthology.orchestra.run.smithi194.stdout:Summary : Support tools for GRUB. 2024-04-10T22:45:00.947 INFO:teuthology.orchestra.run.smithi194.stdout:Description : 2024-04-10T22:45:00.947 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-04-10T22:45:00.947 INFO:teuthology.orchestra.run.smithi194.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-10T22:45:00.947 INFO:teuthology.orchestra.run.smithi194.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-10T22:45:00.947 INFO:teuthology.orchestra.run.smithi194.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-10T22:45:00.947 INFO:teuthology.orchestra.run.smithi194.stdout:hardware devices. 2024-04-10T22:45:00.947 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-04-10T22:45:00.948 INFO:teuthology.orchestra.run.smithi194.stdout:This subpackage provides tools for support of all platforms. 2024-04-10T22:45:00.948 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-10T22:45:00.949 INFO:teuthology.task.kernel:Updating grub on smithi194 to boot 4.18.0-552.el8.x86_64 2024-04-10T22:45:00.949 DEBUG:teuthology.orchestra.run.smithi194:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-10T22:45:02.534 INFO:teuthology.orchestra.run.smithi194.stderr:Generating grub configuration file ... 2024-04-10T22:45:04.677 INFO:teuthology.orchestra.run.smithi194.stderr:done 2024-04-10T22:45:04.679 DEBUG:teuthology.orchestra.run.smithi194:> mktemp 2024-04-10T22:45:04.702 INFO:teuthology.orchestra.run.smithi194.stdout:/tmp/tmp.jxqqFEKNy5 2024-04-10T22:45:04.702 DEBUG:teuthology.orchestra.run.smithi194:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.jxqqFEKNy5 2024-04-10T22:45:04.786 DEBUG:teuthology.orchestra.run.smithi194:> sudo chmod 0666 /tmp/tmp.jxqqFEKNy5 2024-04-10T22:45:04.931 DEBUG:teuthology.orchestra.remote:smithi194:/tmp/tmp.jxqqFEKNy5 is 6KB 2024-04-10T22:45:04.979 DEBUG:teuthology.orchestra.run.smithi194:> rm -fr /tmp/tmp.jxqqFEKNy5 2024-04-10T22:45:04.996 DEBUG:teuthology.orchestra.run.smithi194:> sudo /bin/ls /boot/loader/entries || true 2024-04-10T22:45:05.078 INFO:teuthology.orchestra.run.smithi194.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2024-04-10T22:45:05.078 INFO:teuthology.orchestra.run.smithi194.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2024-04-10T22:45:05.078 INFO:teuthology.orchestra.run.smithi194.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2024-04-10T22:45:05.078 INFO:teuthology.orchestra.run.smithi194.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64.conf 2024-04-10T22:45:05.079 DEBUG:teuthology.orchestra.run.smithi194:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64 2024-04-10T22:45:05.216 DEBUG:teuthology.orchestra.run.smithi194:> sudo shutdown -r now 2024-04-10T22:45:35.219 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-04-10T22:45:35.220 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi194.front.sepia.ceph.com' 2024-04-10T22:45:35.221 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi194.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:46:01.989 INFO:teuthology.orchestra.run.smithi032.stdout:Extra Packages for Enterprise Linux 124 kB/s | 16 MB 02:14 2024-04-10T22:46:05.987 INFO:teuthology.orchestra.run.smithi032.stdout:lab-extras 455 kB/s | 24 kB 00:00 2024-04-10T22:46:06.139 INFO:teuthology.orchestra.run.smithi032.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:46:09.093 INFO:teuthology.orchestra.run.smithi032.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2024-04-10T22:46:09.093 INFO:teuthology.orchestra.run.smithi032.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2024-04-10T22:46:09.190 INFO:teuthology.orchestra.run.smithi032.stdout:Dependencies resolved. 2024-04-10T22:46:09.190 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-04-10T22:46:09.190 INFO:teuthology.orchestra.run.smithi032.stdout: Package Architecture Version Repository Size 2024-04-10T22:46:09.190 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-04-10T22:46:09.190 INFO:teuthology.orchestra.run.smithi032.stdout:Installing: 2024-04-10T22:46:09.191 INFO:teuthology.orchestra.run.smithi032.stdout: kernel x86_64 4.18.0-552.el8 baseos 10 M 2024-04-10T22:46:09.191 INFO:teuthology.orchestra.run.smithi032.stdout:Installing dependencies: 2024-04-10T22:46:09.191 INFO:teuthology.orchestra.run.smithi032.stdout: kernel-core x86_64 4.18.0-552.el8 baseos 44 M 2024-04-10T22:46:09.191 INFO:teuthology.orchestra.run.smithi032.stdout: kernel-modules x86_64 4.18.0-552.el8 baseos 36 M 2024-04-10T22:46:09.191 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-10T22:46:09.191 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction Summary 2024-04-10T22:46:09.191 INFO:teuthology.orchestra.run.smithi032.stdout:================================================================================ 2024-04-10T22:46:09.191 INFO:teuthology.orchestra.run.smithi032.stdout:Install 3 Packages 2024-04-10T22:46:09.191 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-10T22:46:09.192 INFO:teuthology.orchestra.run.smithi032.stdout:Total download size: 90 M 2024-04-10T22:46:09.192 INFO:teuthology.orchestra.run.smithi032.stdout:Installed size: 96 M 2024-04-10T22:46:09.192 INFO:teuthology.orchestra.run.smithi032.stdout:Downloading Packages: 2024-04-10T22:46:35.251 DEBUG:teuthology.orchestra.remote:timed out 2024-04-10T22:46:44.252 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi194.front.sepia.ceph.com' 2024-04-10T22:46:44.253 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi194.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:46:44.592 DEBUG:teuthology.orchestra.run.smithi194:> true 2024-04-10T22:46:44.853 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi194.front.sepia.ceph.com' 2024-04-10T22:46:44.853 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "4.18.0-552.el8.x86_64"... 2024-04-10T22:46:44.853 DEBUG:teuthology.orchestra.run.smithi194:> uname -r 2024-04-10T22:46:44.870 INFO:teuthology.orchestra.run.smithi194.stdout:4.18.0-552.el8.x86_64 2024-04-10T22:46:44.870 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-552.el8.x86_64 vs 4.18.0-552.el8.x86_64 2024-04-10T22:46:44.870 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-10T22:46:44.870 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-10T22:46:45.871 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-04-10T22:46:45.871 DEBUG:teuthology.orchestra.run.smithi194:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-10T22:46:45.953 INFO:teuthology.orchestra.run.smithi194.stdout:ttyS1 2024-04-10T22:46:45.969 DEBUG:teuthology.parallel:result is None 2024-04-10T22:48:21.969 INFO:teuthology.orchestra.run.smithi032.stdout:(1/3): kernel-4.18.0-552.el8.x86_64.rpm 3.7 MB/s | 10 MB 00:02 2024-04-10T22:48:26.016 INFO:teuthology.orchestra.run.smithi032.stdout:(2/3): kernel-core-4.18.0-552.el8.x86_64.rpm 6.4 MB/s | 44 MB 00:06 2024-04-10T22:48:28.424 INFO:teuthology.orchestra.run.smithi032.stdout:(3/3): kernel-modules-4.18.0-552.el8.x86_64.rpm 4.0 MB/s | 36 MB 00:09 2024-04-10T22:48:28.424 INFO:teuthology.orchestra.run.smithi032.stdout:-------------------------------------------------------------------------------- 2024-04-10T22:48:28.424 INFO:teuthology.orchestra.run.smithi032.stdout:Total 665 kB/s | 90 MB 02:19 2024-04-10T22:48:29.140 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction check 2024-04-10T22:48:29.636 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction check succeeded. 2024-04-10T22:48:29.636 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction test 2024-04-10T22:48:30.956 INFO:teuthology.orchestra.run.smithi032.stdout:Transaction test succeeded. 2024-04-10T22:48:30.988 INFO:teuthology.orchestra.run.smithi032.stdout:Running transaction 2024-04-10T22:48:34.004 INFO:teuthology.orchestra.run.smithi032.stdout: Preparing : 1/1 2024-04-10T22:48:34.600 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:48:38.383 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:48:38.783 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:48:42.474 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:48:42.801 INFO:teuthology.orchestra.run.smithi032.stdout: Installing : kernel-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:49:06.386 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:49:06.410 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:49:06.779 INFO:teuthology.orchestra.run.smithi032.stdout: Running scriptlet: kernel-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:49:06.779 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : kernel-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:49:06.779 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : kernel-core-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:49:07.374 INFO:teuthology.orchestra.run.smithi032.stdout: Verifying : kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:49:07.374 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-10T22:49:07.374 INFO:teuthology.orchestra.run.smithi032.stdout:Installed: 2024-04-10T22:49:07.374 INFO:teuthology.orchestra.run.smithi032.stdout: kernel-4.18.0-552.el8.x86_64 kernel-core-4.18.0-552.el8.x86_64 2024-04-10T22:49:07.374 INFO:teuthology.orchestra.run.smithi032.stdout: kernel-modules-4.18.0-552.el8.x86_64 2024-04-10T22:49:07.374 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-10T22:49:07.374 INFO:teuthology.orchestra.run.smithi032.stdout:Complete! 2024-04-10T22:49:07.501 DEBUG:teuthology.orchestra.run.smithi032:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:49:07.771 INFO:teuthology.orchestra.run.smithi032.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-10T22:49:07.771 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-10T22:49:07.771 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-552.el8.x86_64 2024-04-10T22:49:07.771 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-10T22:49:07.771 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-10T22:49:07.771 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-04-10T22:49:07.771 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi032.front.sepia.ceph.com, path=None, version=distro) 2024-04-10T22:49:07.772 DEBUG:teuthology.orchestra.run.smithi032:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:49:07.992 INFO:teuthology.orchestra.run.smithi032.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-10T22:49:07.992 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-10T22:49:07.992 DEBUG:teuthology.orchestra.run.smithi032:> sudo rpm -qi grub2-tools 2024-04-10T22:49:08.027 INFO:teuthology.orchestra.run.smithi032.stdout:Name : grub2-tools 2024-04-10T22:49:08.027 INFO:teuthology.orchestra.run.smithi032.stdout:Epoch : 1 2024-04-10T22:49:08.028 INFO:teuthology.orchestra.run.smithi032.stdout:Version : 2.02 2024-04-10T22:49:08.028 INFO:teuthology.orchestra.run.smithi032.stdout:Release : 129.el8 2024-04-10T22:49:08.028 INFO:teuthology.orchestra.run.smithi032.stdout:Architecture: x86_64 2024-04-10T22:49:08.028 INFO:teuthology.orchestra.run.smithi032.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2024-04-10T22:49:08.028 INFO:teuthology.orchestra.run.smithi032.stdout:Group : System Environment/Base 2024-04-10T22:49:08.028 INFO:teuthology.orchestra.run.smithi032.stdout:Size : 9265586 2024-04-10T22:49:08.028 INFO:teuthology.orchestra.run.smithi032.stdout:License : GPLv3+ 2024-04-10T22:49:08.028 INFO:teuthology.orchestra.run.smithi032.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2024-04-10T22:49:08.028 INFO:teuthology.orchestra.run.smithi032.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2024-04-10T22:49:08.028 INFO:teuthology.orchestra.run.smithi032.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2024-04-10T22:49:08.028 INFO:teuthology.orchestra.run.smithi032.stdout:Build Host : kbuilder.bsys.centos.org 2024-04-10T22:49:08.028 INFO:teuthology.orchestra.run.smithi032.stdout:Relocations : (not relocatable) 2024-04-10T22:49:08.028 INFO:teuthology.orchestra.run.smithi032.stdout:Packager : CentOS BuildSystem 2024-04-10T22:49:08.028 INFO:teuthology.orchestra.run.smithi032.stdout:Vendor : CentOS 2024-04-10T22:49:08.028 INFO:teuthology.orchestra.run.smithi032.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-10T22:49:08.029 INFO:teuthology.orchestra.run.smithi032.stdout:Summary : Support tools for GRUB. 2024-04-10T22:49:08.029 INFO:teuthology.orchestra.run.smithi032.stdout:Description : 2024-04-10T22:49:08.029 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-10T22:49:08.029 INFO:teuthology.orchestra.run.smithi032.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-10T22:49:08.029 INFO:teuthology.orchestra.run.smithi032.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-10T22:49:08.029 INFO:teuthology.orchestra.run.smithi032.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-10T22:49:08.029 INFO:teuthology.orchestra.run.smithi032.stdout:hardware devices. 2024-04-10T22:49:08.029 INFO:teuthology.orchestra.run.smithi032.stdout: 2024-04-10T22:49:08.029 INFO:teuthology.orchestra.run.smithi032.stdout:This subpackage provides tools for support of all platforms. 2024-04-10T22:49:08.030 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-10T22:49:08.030 INFO:teuthology.task.kernel:Updating grub on smithi032 to boot 4.18.0-552.el8.x86_64 2024-04-10T22:49:08.030 DEBUG:teuthology.orchestra.run.smithi032:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-10T22:49:09.501 INFO:teuthology.orchestra.run.smithi032.stderr:Generating grub configuration file ... 2024-04-10T22:49:11.213 INFO:teuthology.orchestra.run.smithi032.stderr:done 2024-04-10T22:49:11.214 DEBUG:teuthology.orchestra.run.smithi032:> mktemp 2024-04-10T22:49:11.247 INFO:teuthology.orchestra.run.smithi032.stdout:/tmp/tmp.2eGNvGlduY 2024-04-10T22:49:11.247 DEBUG:teuthology.orchestra.run.smithi032:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.2eGNvGlduY 2024-04-10T22:49:11.320 DEBUG:teuthology.orchestra.run.smithi032:> sudo chmod 0666 /tmp/tmp.2eGNvGlduY 2024-04-10T22:49:11.461 DEBUG:teuthology.orchestra.remote:smithi032:/tmp/tmp.2eGNvGlduY is 6KB 2024-04-10T22:49:11.510 DEBUG:teuthology.orchestra.run.smithi032:> rm -fr /tmp/tmp.2eGNvGlduY 2024-04-10T22:49:11.528 DEBUG:teuthology.orchestra.run.smithi032:> sudo /bin/ls /boot/loader/entries || true 2024-04-10T22:49:11.629 INFO:teuthology.orchestra.run.smithi032.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2024-04-10T22:49:11.630 INFO:teuthology.orchestra.run.smithi032.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2024-04-10T22:49:11.630 INFO:teuthology.orchestra.run.smithi032.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2024-04-10T22:49:11.630 INFO:teuthology.orchestra.run.smithi032.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64.conf 2024-04-10T22:49:11.631 DEBUG:teuthology.orchestra.run.smithi032:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64 2024-04-10T22:49:11.759 DEBUG:teuthology.orchestra.run.smithi032:> sudo shutdown -r now 2024-04-10T22:49:41.761 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-04-10T22:49:41.762 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi032.front.sepia.ceph.com' 2024-04-10T22:49:41.763 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi032.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:50:41.766 DEBUG:teuthology.orchestra.remote:timed out 2024-04-10T22:50:50.767 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi032.front.sepia.ceph.com' 2024-04-10T22:50:50.769 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi032.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:50:51.092 DEBUG:teuthology.orchestra.run.smithi032:> true 2024-04-10T22:50:51.384 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi032.front.sepia.ceph.com' 2024-04-10T22:50:51.384 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "4.18.0-552.el8.x86_64"... 2024-04-10T22:50:51.384 DEBUG:teuthology.orchestra.run.smithi032:> uname -r 2024-04-10T22:50:51.404 INFO:teuthology.orchestra.run.smithi032.stdout:4.18.0-552.el8.x86_64 2024-04-10T22:50:51.404 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-552.el8.x86_64 vs 4.18.0-552.el8.x86_64 2024-04-10T22:50:51.404 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-10T22:50:51.404 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-10T22:50:52.405 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-04-10T22:50:52.405 DEBUG:teuthology.orchestra.run.smithi032:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-10T22:50:52.480 INFO:teuthology.orchestra.run.smithi032.stdout:ttyS1 2024-04-10T22:50:52.494 DEBUG:teuthology.parallel:result is None 2024-04-10T22:50:52.494 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-10T22:50:52.502 INFO:teuthology.task.internal:Creating test directory... 2024-04-10T22:50:52.502 DEBUG:teuthology.orchestra.run.smithi032:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-10T22:50:52.505 DEBUG:teuthology.orchestra.run.smithi194:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-10T22:50:52.529 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-10T22:50:52.535 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-10T22:50:52.544 INFO:teuthology.task.internal:Creating archive directory... 2024-04-10T22:50:52.544 DEBUG:teuthology.orchestra.run.smithi032:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-10T22:50:52.566 DEBUG:teuthology.orchestra.run.smithi194:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-10T22:50:52.613 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-10T22:50:52.623 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-10T22:50:52.623 DEBUG:teuthology.orchestra.run.smithi032:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-10T22:50:52.650 DEBUG:teuthology.orchestra.run.smithi194:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-10T22:50:52.682 INFO:teuthology.orchestra.run.smithi032.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:50:52.690 INFO:teuthology.orchestra.run.smithi194.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:50:52.693 INFO:teuthology.orchestra.run.smithi032.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:50:52.701 INFO:teuthology.orchestra.run.smithi194.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:50:52.702 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-10T22:50:52.710 INFO:teuthology.task.internal:Configuring sudo... 2024-04-10T22:50:52.710 DEBUG:teuthology.orchestra.run.smithi032:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-10T22:50:52.738 DEBUG:teuthology.orchestra.run.smithi194:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-10T22:50:52.774 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-10T22:50:52.783 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-10T22:50:52.783 DEBUG:teuthology.orchestra.run.smithi032:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-10T22:50:52.808 DEBUG:teuthology.orchestra.run.smithi194:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-10T22:50:52.833 DEBUG:teuthology.orchestra.run.smithi032:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:50:52.915 DEBUG:teuthology.orchestra.run.smithi032:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:50:52.965 DEBUG:teuthology.orchestra.run.smithi032:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:50:53.054 DEBUG:teuthology.orchestra.run.smithi032:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:50:53.096 DEBUG:teuthology.orchestra.run.smithi032:> set -ex 2024-04-10T22:50:53.097 DEBUG:teuthology.orchestra.run.smithi032:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-10T22:50:53.165 DEBUG:teuthology.orchestra.run.smithi194:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:50:53.214 DEBUG:teuthology.orchestra.run.smithi194:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:50:53.281 DEBUG:teuthology.orchestra.run.smithi194:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:50:53.327 DEBUG:teuthology.orchestra.run.smithi194:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:50:53.371 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-04-10T22:50:53.371 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-10T22:50:53.439 DEBUG:teuthology.orchestra.run.smithi032:> sudo service rsyslog restart 2024-04-10T22:50:53.442 DEBUG:teuthology.orchestra.run.smithi194:> sudo service rsyslog restart 2024-04-10T22:50:53.501 INFO:teuthology.orchestra.run.smithi032.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:50:53.541 INFO:teuthology.orchestra.run.smithi194.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:50:53.823 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-10T22:50:53.832 INFO:teuthology.task.internal:Starting timer... 2024-04-10T22:50:53.832 INFO:teuthology.run_tasks:Running task pcp... 2024-04-10T22:50:53.846 INFO:teuthology.run_tasks:Running task selinux... 2024-04-10T22:50:53.862 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-04-10T22:50:53.862 DEBUG:teuthology.orchestra.run.smithi032:> sudo service auditd rotate 2024-04-10T22:50:53.963 INFO:teuthology.orchestra.run.smithi032.stdout:Rotating logs: 2024-04-10T22:50:53.965 DEBUG:teuthology.orchestra.run.smithi194:> sudo service auditd rotate 2024-04-10T22:50:54.040 INFO:teuthology.orchestra.run.smithi194.stdout:Rotating logs: 2024-04-10T22:50:54.041 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-10T22:50:54.041 DEBUG:teuthology.orchestra.run.smithi032:> /usr/sbin/getenforce 2024-04-10T22:50:54.094 INFO:teuthology.orchestra.run.smithi032.stdout:Permissive 2024-04-10T22:50:54.094 DEBUG:teuthology.orchestra.run.smithi194:> /usr/sbin/getenforce 2024-04-10T22:50:54.127 INFO:teuthology.orchestra.run.smithi194.stdout:Permissive 2024-04-10T22:50:54.127 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi032.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi194.front.sepia.ceph.com': 'permissive'} 2024-04-10T22:50:54.127 DEBUG:teuthology.orchestra.run.smithi032:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:50:54.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:50:54.157 DEBUG:teuthology.orchestra.run.smithi194:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:50:54.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:50:54.193 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-10T22:50:54.194 DEBUG:teuthology.orchestra.run.smithi032:> sudo /usr/sbin/setenforce permissive 2024-04-10T22:50:54.227 DEBUG:teuthology.orchestra.run.smithi194:> sudo /usr/sbin/setenforce permissive 2024-04-10T22:50:54.260 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-10T22:50:54.270 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-10T22:50:54.286 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-10T22:50:54.287 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi032.front.sepia.ceph.com,smithi194.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-10T23:04:16.011 INFO:teuthology.task.ansible:Archiving ansible failure log at: /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650189/ansible_failures.yaml 2024-04-10T23:04:16.013 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/__init__.py", line 123, in __enter__ self.begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 508, in begin super(CephLab, self).begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 342, in begin self.execute_playbook() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 374, in execute_playbook self._handle_failure(command, status) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 402, in _handle_failure raise AnsibleFailedError(failures) teuthology.exceptions.AnsibleFailedError: [',', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at'] 2024-04-10T23:04:16.245 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=356cc65258854b81b6491401e01a5d17 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/__init__.py", line 123, in __enter__ self.begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 508, in begin super(CephLab, self).begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 342, in begin self.execute_playbook() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 374, in execute_playbook self._handle_failure(command, status) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 402, in _handle_failure raise AnsibleFailedError(failures) teuthology.exceptions.AnsibleFailedError: [',', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at'] 2024-04-10T23:04:16.247 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-10T23:04:16.256 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-10T23:04:16.256 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-10T23:04:16.274 DEBUG:teuthology.orchestra.run.smithi032:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-10T23:04:16.354 DEBUG:teuthology.orchestra.run.smithi194:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-10T23:04:16.419 DEBUG:teuthology.orchestra.run.smithi032:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T23:04:16.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T23:04:16.451 DEBUG:teuthology.orchestra.run.smithi194:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T23:04:16.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T23:04:16.483 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-10T23:04:16.491 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-10T23:04:16.550 INFO:teuthology.task.internal:Duration was 802.718346 seconds 2024-04-10T23:04:16.550 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-10T23:04:16.563 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-10T23:04:16.563 DEBUG:teuthology.orchestra.run.smithi032:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-10T23:04:16.566 DEBUG:teuthology.orchestra.run.smithi194:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-10T23:04:16.614 INFO:teuthology.orchestra.run.smithi194.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T23:04:16.615 INFO:teuthology.orchestra.run.smithi032.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T23:04:17.074 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-10T23:04:17.075 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi032.front.sepia.ceph.com 2024-04-10T23:04:17.075 DEBUG:teuthology.orchestra.run.smithi032:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-10T23:04:17.103 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi194.front.sepia.ceph.com 2024-04-10T23:04:17.103 DEBUG:teuthology.orchestra.run.smithi194:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-10T23:04:17.130 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-10T23:04:17.130 DEBUG:teuthology.orchestra.run.smithi032:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-10T23:04:17.146 DEBUG:teuthology.orchestra.run.smithi194:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-10T23:04:17.214 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-10T23:04:17.214 DEBUG:teuthology.orchestra.run.smithi032:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-10T23:04:17.217 DEBUG:teuthology.orchestra.run.smithi194:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-10T23:04:17.319 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-10T23:04:17.329 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-10T23:04:17.329 DEBUG:teuthology.orchestra.run.smithi032:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-10T23:04:17.358 DEBUG:teuthology.orchestra.run.smithi194:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-10T23:04:17.385 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-10T23:04:17.395 DEBUG:teuthology.orchestra.run.smithi032:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-10T23:04:17.400 DEBUG:teuthology.orchestra.run.smithi194:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-10T23:04:17.426 INFO:teuthology.orchestra.run.smithi032.stdout:kernel.core_pattern = core 2024-04-10T23:04:17.453 INFO:teuthology.orchestra.run.smithi194.stdout:kernel.core_pattern = core 2024-04-10T23:04:17.472 DEBUG:teuthology.orchestra.run.smithi032:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-10T23:04:17.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T23:04:17.522 DEBUG:teuthology.orchestra.run.smithi194:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-10T23:04:17.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T23:04:17.539 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-10T23:04:17.552 INFO:teuthology.task.internal:Transferring archived files... 2024-04-10T23:04:17.554 DEBUG:teuthology.misc:Transferring archived files from smithi032:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650189/remote/smithi032 2024-04-10T23:04:17.555 DEBUG:teuthology.orchestra.run.smithi032:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-10T23:04:17.623 DEBUG:teuthology.misc:Transferring archived files from smithi194:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650189/remote/smithi194 2024-04-10T23:04:17.624 DEBUG:teuthology.orchestra.run.smithi194:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-10T23:04:17.666 INFO:teuthology.task.internal:Removing archive directory... 2024-04-10T23:04:17.667 DEBUG:teuthology.orchestra.run.smithi032:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-10T23:04:17.669 DEBUG:teuthology.orchestra.run.smithi194:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-10T23:04:17.719 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-10T23:04:17.730 INFO:teuthology.task.internal:Not uploading archives. 2024-04-10T23:04:17.730 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-10T23:04:17.739 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-10T23:04:17.740 DEBUG:teuthology.orchestra.run.smithi032:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-10T23:04:17.742 DEBUG:teuthology.orchestra.run.smithi194:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-10T23:04:17.758 INFO:teuthology.orchestra.run.smithi032.stdout: 525355 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 10 23:04 /home/ubuntu/cephtest 2024-04-10T23:04:17.776 INFO:teuthology.orchestra.run.smithi194.stdout: 393224 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 10 23:04 /home/ubuntu/cephtest 2024-04-10T23:04:17.777 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-10T23:04:17.787 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-10T23:04:17.806 INFO:teuthology.nuke:Checking targets against current locks 2024-04-10T23:04:17.840 DEBUG:teuthology.nuke:shortname: smithi032 2024-04-10T23:04:17.841 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T23:04:17.881 DEBUG:teuthology.nuke:shortname: smithi194 2024-04-10T23:04:17.881 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T23:04:17.895 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi032.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650189', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:19.221626', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFMfiV2F3C5RwI85TIdaj6uPGhdnQMnccTu7MUJZMX1rG3qn976HpbwkSLmeauSIKOXQRWx8RTUQRbuXJTaId4U='} 2024-04-10T23:04:17.927 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi194.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650189', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:19.223105', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGh/6nZGaFxmOMax7pKbRq6YsdK+3vj8be0kmr3iGQEI16T9Isv6/VLGkIumJp4FfwCTO6hQaaIOW0iUZNCeIrc='} 2024-04-10T23:04:17.990 INFO:teuthology.orchestra.console.smithi032:Power off 2024-04-10T23:04:17.990 DEBUG:teuthology.orchestra.console.smithi032:pexpect command: ipmitool -H smithi032.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-10T23:04:18.024 INFO:teuthology.orchestra.console.smithi194:Power off 2024-04-10T23:04:18.025 DEBUG:teuthology.orchestra.console.smithi194:pexpect command: ipmitool -H smithi194.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-10T23:04:18.042 DEBUG:teuthology.orchestra.console.smithi032:power off output: Chassis Power Control: Down/Off 2024-04-10T23:04:18.043 DEBUG:teuthology.orchestra.console.smithi032:pexpect command: ipmitool -H smithi032.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:04:18.077 DEBUG:teuthology.orchestra.console.smithi194:power off output: Chassis Power Control: Down/Off 2024-04-10T23:04:18.078 DEBUG:teuthology.orchestra.console.smithi194:pexpect command: ipmitool -H smithi194.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:04:18.098 DEBUG:teuthology.orchestra.console.smithi032:check power output: Chassis Power is on 2024-04-10T23:04:18.130 DEBUG:teuthology.orchestra.console.smithi194:check power output: Chassis Power is on 2024-04-10T23:04:22.099 DEBUG:teuthology.orchestra.console.smithi032:pexpect command: ipmitool -H smithi032.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:04:22.131 DEBUG:teuthology.orchestra.console.smithi194:pexpect command: ipmitool -H smithi194.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:04:22.210 DEBUG:teuthology.orchestra.console.smithi032:check power output: Chassis Power is on 2024-04-10T23:04:22.241 DEBUG:teuthology.orchestra.console.smithi194:check power output: Chassis Power is on 2024-04-10T23:04:26.212 DEBUG:teuthology.orchestra.console.smithi032:pexpect command: ipmitool -H smithi032.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:04:26.243 DEBUG:teuthology.orchestra.console.smithi194:pexpect command: ipmitool -H smithi194.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:04:26.323 DEBUG:teuthology.orchestra.console.smithi032:check power output: Chassis Power is off 2024-04-10T23:04:26.354 DEBUG:teuthology.orchestra.console.smithi194:check power output: Chassis Power is off 2024-04-10T23:04:26.425 INFO:teuthology.orchestra.console.smithi032:Power off completed 2024-04-10T23:04:26.455 INFO:teuthology.orchestra.console.smithi194:Power off completed 2024-04-10T23:04:26.561 INFO:teuthology.run:Summary data: description: upgrade:pacific-x/stress-split/{0-distro/centos_8.stream_container_tools_crun 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} duration: 802.7183458805084 failure_reason: '['','', ''Bugzilla. In case of problems, contact the owner of this repository.'', ''Enabling a Copr repository. Please note that this repository is not part'', ''Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable'', ''Please do not file bug reports about these packages in Fedora'', ''The Fedora Project does not exercise any power over the contents of'', ''and packages are not held to any quality or security level.'', ''of the main distribution, and quality may vary.'', ''this repository beyond the rules outlined in the Copr FAQ at'']' owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=356cc65258854b81b6491401e01a5d17 status: dead success: false 2024-04-10T23:04:26.562 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-10T23:04:26.661 INFO:teuthology.run:DEAD