2024-04-10T22:23:04.705 INFO:root:teuthology version: 0.0.1.dev267+g6c63784 2024-04-10T22:23:04.706 DEBUG:teuthology.run:Teuthology command: teuthology --name teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi --owner scheduled_teuthology@teuthology --archive /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650169 --verbose --description upgrade:pacific-x/stress-split/{0-distro/centos_8.stream_container_tools_crun 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} -- /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650169/orig.config.yaml 2024-04-10T22:23:04.759 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-10T22:23:04.846 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650169 branch: reef description: upgrade:pacific-x/stress-split/{0-distro/centos_8.stream_container_tools_crun 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} email: ceph-qa@ceph.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: pacific clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' first_in_suite: false job_id: '7650169' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run basic cls tests for rbd ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run basic import/export cli tests for rbd ' name: teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 8.stream overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: reef sha1: 4f70a9a90881107996c8b7a3e89ba8b32624713c owner: scheduled_teuthology@teuthology priority: 830 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' seed: 4604 sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: pacific clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 0/1 suite: upgrade:pacific-x suite_branch: reef suite_path: /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 4f70a9a90881107996c8b7a3e89ba8b32624713c targets: smithi088.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBIHgHKExPoXrhhBorDqIfE8KUwKOaIzeUXMjkWH25sbPe4rcItwywAWJhGVhgRocA8ezsdJwTfhX7Zb5BuxqImQ= smithi132.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBG4qGLVgVzTPUpDraSjMUYExRwLyaPdU2mp2Ux6Qq6hJogoS7kC5Keg4tuThvW2GuEvgqvfNp5u91EuAAcLoH4I= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - sudo sed -i 's/runtime = "runc"/#runtime = "runc"/g' /usr/share/containers/containers.conf - sudo sed -i 's/#runtime = "crun"/runtime = "crun"/g' /usr/share/containers/containers.conf - install: branch: pacific exclude_packages: - ceph-volume - cephadm: cephadm_branch: pacific cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:pacific - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 6c637841c215537a4502385240412f1966e0faab timestamp: 2024-04-10_01:08:08 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-04-10T22:23:04.846 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa; will attempt to use it 2024-04-10T22:23:04.846 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa/tasks 2024-04-10T22:23:04.847 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-10T22:23:04.848 INFO:teuthology.task.internal:Checking packages... 2024-04-10T22:23:04.869 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'd540ebaca6b131a1dd560e7f69e024b133bbaa42' 2024-04-10T22:23:04.869 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-10T22:23:04.869 INFO:teuthology.packaging:ref: None 2024-04-10T22:23:04.869 INFO:teuthology.packaging:tag: None 2024-04-10T22:23:04.869 INFO:teuthology.packaging:branch: reef 2024-04-10T22:23:04.869 INFO:teuthology.packaging:sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 2024-04-10T22:23:04.869 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef 2024-04-10T22:23:05.232 INFO:teuthology.task.internal:Found packages for ceph version 18.2.2-1189.gc96c8d47 2024-04-10T22:23:05.234 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-10T22:23:05.240 INFO:teuthology.task.internal:no buildpackages task found 2024-04-10T22:23:05.240 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-10T22:23:05.262 INFO:teuthology.task.internal:Saving configuration 2024-04-10T22:23:05.281 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-10T22:23:05.287 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T22:23:05.313 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi088.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650169', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:15:48.871204', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBIHgHKExPoXrhhBorDqIfE8KUwKOaIzeUXMjkWH25sbPe4rcItwywAWJhGVhgRocA8ezsdJwTfhX7Zb5BuxqImQ='} 2024-04-10T22:23:05.336 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi132.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650169', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:15:48.872821', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBG4qGLVgVzTPUpDraSjMUYExRwLyaPdU2mp2Ux6Qq6hJogoS7kC5Keg4tuThvW2GuEvgqvfNp5u91EuAAcLoH4I='} 2024-04-10T22:23:05.336 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-10T22:23:05.343 INFO:teuthology.task.internal:roles: ubuntu@smithi088.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-04-10T22:23:05.343 INFO:teuthology.task.internal:roles: ubuntu@smithi132.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-04-10T22:23:05.343 INFO:teuthology.run_tasks:Running task console_log... 2024-04-10T22:23:05.408 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f5fd32df0d0>, signals=[15]) 2024-04-10T22:23:05.409 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-10T22:23:05.415 INFO:teuthology.task.internal:Opening connections... 2024-04-10T22:23:05.415 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi088.front.sepia.ceph.com 2024-04-10T22:23:05.417 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi088.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:23:05.491 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi132.front.sepia.ceph.com 2024-04-10T22:23:05.491 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi132.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:23:05.560 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-10T22:23:05.566 DEBUG:teuthology.orchestra.run.smithi088:> uname -m 2024-04-10T22:23:05.588 INFO:teuthology.orchestra.run.smithi088.stdout:x86_64 2024-04-10T22:23:05.588 DEBUG:teuthology.orchestra.run.smithi088:> cat /etc/os-release 2024-04-10T22:23:05.649 INFO:teuthology.orchestra.run.smithi088.stdout:NAME="CentOS Stream" 2024-04-10T22:23:05.649 INFO:teuthology.orchestra.run.smithi088.stdout:VERSION="8" 2024-04-10T22:23:05.649 INFO:teuthology.orchestra.run.smithi088.stdout:ID="centos" 2024-04-10T22:23:05.649 INFO:teuthology.orchestra.run.smithi088.stdout:ID_LIKE="rhel fedora" 2024-04-10T22:23:05.649 INFO:teuthology.orchestra.run.smithi088.stdout:VERSION_ID="8" 2024-04-10T22:23:05.649 INFO:teuthology.orchestra.run.smithi088.stdout:PLATFORM_ID="platform:el8" 2024-04-10T22:23:05.649 INFO:teuthology.orchestra.run.smithi088.stdout:PRETTY_NAME="CentOS Stream 8" 2024-04-10T22:23:05.649 INFO:teuthology.orchestra.run.smithi088.stdout:ANSI_COLOR="0;31" 2024-04-10T22:23:05.649 INFO:teuthology.orchestra.run.smithi088.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2024-04-10T22:23:05.649 INFO:teuthology.orchestra.run.smithi088.stdout:HOME_URL="https://centos.org/" 2024-04-10T22:23:05.649 INFO:teuthology.orchestra.run.smithi088.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-10T22:23:05.649 INFO:teuthology.orchestra.run.smithi088.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2024-04-10T22:23:05.650 INFO:teuthology.orchestra.run.smithi088.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-10T22:23:05.650 INFO:teuthology.lock.ops:Updating smithi088.front.sepia.ceph.com on lock server 2024-04-10T22:23:05.673 DEBUG:teuthology.orchestra.run.smithi132:> uname -m 2024-04-10T22:23:05.695 INFO:teuthology.orchestra.run.smithi132.stdout:x86_64 2024-04-10T22:23:05.695 DEBUG:teuthology.orchestra.run.smithi132:> cat /etc/os-release 2024-04-10T22:23:05.757 INFO:teuthology.orchestra.run.smithi132.stdout:NAME="CentOS Stream" 2024-04-10T22:23:05.757 INFO:teuthology.orchestra.run.smithi132.stdout:VERSION="8" 2024-04-10T22:23:05.757 INFO:teuthology.orchestra.run.smithi132.stdout:ID="centos" 2024-04-10T22:23:05.757 INFO:teuthology.orchestra.run.smithi132.stdout:ID_LIKE="rhel fedora" 2024-04-10T22:23:05.757 INFO:teuthology.orchestra.run.smithi132.stdout:VERSION_ID="8" 2024-04-10T22:23:05.757 INFO:teuthology.orchestra.run.smithi132.stdout:PLATFORM_ID="platform:el8" 2024-04-10T22:23:05.757 INFO:teuthology.orchestra.run.smithi132.stdout:PRETTY_NAME="CentOS Stream 8" 2024-04-10T22:23:05.757 INFO:teuthology.orchestra.run.smithi132.stdout:ANSI_COLOR="0;31" 2024-04-10T22:23:05.757 INFO:teuthology.orchestra.run.smithi132.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2024-04-10T22:23:05.758 INFO:teuthology.orchestra.run.smithi132.stdout:HOME_URL="https://centos.org/" 2024-04-10T22:23:05.758 INFO:teuthology.orchestra.run.smithi132.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-10T22:23:05.758 INFO:teuthology.orchestra.run.smithi132.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2024-04-10T22:23:05.758 INFO:teuthology.orchestra.run.smithi132.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-10T22:23:05.758 INFO:teuthology.lock.ops:Updating smithi132.front.sepia.ceph.com on lock server 2024-04-10T22:23:05.781 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-10T22:23:05.791 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-10T22:23:05.796 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-10T22:23:05.796 DEBUG:teuthology.orchestra.run.smithi088:> test '!' -e /home/ubuntu/cephtest 2024-04-10T22:23:05.798 DEBUG:teuthology.orchestra.run.smithi132:> test '!' -e /home/ubuntu/cephtest 2024-04-10T22:23:05.818 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-10T22:23:05.824 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-10T22:23:05.824 DEBUG:teuthology.orchestra.run.smithi088:> test -z $(ls -A /var/lib/ceph) 2024-04-10T22:23:05.859 DEBUG:teuthology.orchestra.run.smithi132:> test -z $(ls -A /var/lib/ceph) 2024-04-10T22:23:05.896 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-10T22:23:05.951 INFO:teuthology.run_tasks:Running task kernel... 2024-04-10T22:23:05.964 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-04-10T22:23:05.964 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-04-10T22:23:05.964 DEBUG:teuthology.orchestra.run.smithi088:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-10T22:23:05.964 DEBUG:teuthology.orchestra.run.smithi132:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-10T22:23:05.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:23:05.984 DEBUG:teuthology.orchestra.run.smithi132:> uname -r 2024-04-10T22:23:05.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:23:05.985 DEBUG:teuthology.orchestra.run.smithi088:> uname -r 2024-04-10T22:23:06.043 INFO:teuthology.orchestra.run.smithi132.stdout:4.18.0-489.el8.x86_64 2024-04-10T22:23:06.044 INFO:teuthology.task.kernel:Running kernel on smithi132: 4.18.0-489.el8.x86_64 2024-04-10T22:23:06.044 DEBUG:teuthology.orchestra.run.smithi132:> sudo yum install -y kernel 2024-04-10T22:23:06.046 INFO:teuthology.orchestra.run.smithi088.stdout:4.18.0-489.el8.x86_64 2024-04-10T22:23:06.046 INFO:teuthology.task.kernel:Running kernel on smithi088: 4.18.0-489.el8.x86_64 2024-04-10T22:23:06.046 DEBUG:teuthology.orchestra.run.smithi088:> sudo yum install -y kernel 2024-04-10T22:25:18.751 INFO:teuthology.orchestra.run.smithi132.stdout:CentOS Stream 8 - AppStream 219 kB/s | 28 MB 02:10 2024-04-10T22:25:20.880 INFO:teuthology.orchestra.run.smithi088.stdout:CentOS Stream 8 - AppStream 216 kB/s | 28 MB 02:12 2024-04-10T22:25:24.934 INFO:teuthology.orchestra.run.smithi132.stdout:CentOS Stream 8 - BaseOS 8.8 MB/s | 10 MB 00:01 2024-04-10T22:25:26.963 INFO:teuthology.orchestra.run.smithi088.stdout:CentOS Stream 8 - BaseOS 9.6 MB/s | 10 MB 00:01 2024-04-10T22:27:37.234 INFO:teuthology.orchestra.run.smithi132.stdout:CentOS Stream 8 - Extras 139 B/s | 18 kB 02:10 2024-04-10T22:27:40.054 INFO:teuthology.orchestra.run.smithi088.stdout:CentOS Stream 8 - Extras 139 B/s | 18 kB 02:11 2024-04-10T22:29:48.450 INFO:teuthology.orchestra.run.smithi132.stdout:CentOS Stream 8 - Extras common packages 60 B/s | 7.7 kB 02:11 2024-04-10T22:29:50.344 INFO:teuthology.orchestra.run.smithi088.stdout:CentOS Stream 8 - Extras common packages 61 B/s | 7.7 kB 02:10 2024-04-10T22:30:28.826 INFO:teuthology.orchestra.run.smithi132.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:40 2024-04-10T22:30:28.827 INFO:teuthology.orchestra.run.smithi132.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:30:28.827 INFO:teuthology.orchestra.run.smithi132.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:30:28.827 INFO:teuthology.orchestra.run.smithi132.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:30:28.926 INFO:teuthology.orchestra.run.smithi088.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:38 2024-04-10T22:30:28.927 INFO:teuthology.orchestra.run.smithi088.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:30:28.927 INFO:teuthology.orchestra.run.smithi088.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:30:28.927 INFO:teuthology.orchestra.run.smithi088.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:30:30.905 INFO:teuthology.orchestra.run.smithi088.stdout:Extra Packages for Enterprise Linux 8.2 MB/s | 16 MB 00:01 2024-04-10T22:30:34.696 INFO:teuthology.orchestra.run.smithi088.stdout:lab-extras 426 kB/s | 24 kB 00:00 2024-04-10T22:30:34.839 INFO:teuthology.orchestra.run.smithi088.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:30:37.676 INFO:teuthology.orchestra.run.smithi088.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2024-04-10T22:30:37.676 INFO:teuthology.orchestra.run.smithi088.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2024-04-10T22:30:37.794 INFO:teuthology.orchestra.run.smithi088.stdout:Dependencies resolved. 2024-04-10T22:30:37.795 INFO:teuthology.orchestra.run.smithi088.stdout:================================================================================ 2024-04-10T22:30:37.795 INFO:teuthology.orchestra.run.smithi088.stdout: Package Architecture Version Repository Size 2024-04-10T22:30:37.795 INFO:teuthology.orchestra.run.smithi088.stdout:================================================================================ 2024-04-10T22:30:37.795 INFO:teuthology.orchestra.run.smithi088.stdout:Installing: 2024-04-10T22:30:37.795 INFO:teuthology.orchestra.run.smithi088.stdout: kernel x86_64 4.18.0-552.el8 baseos 10 M 2024-04-10T22:30:37.795 INFO:teuthology.orchestra.run.smithi088.stdout:Installing dependencies: 2024-04-10T22:30:37.795 INFO:teuthology.orchestra.run.smithi088.stdout: kernel-core x86_64 4.18.0-552.el8 baseos 44 M 2024-04-10T22:30:37.795 INFO:teuthology.orchestra.run.smithi088.stdout: kernel-modules x86_64 4.18.0-552.el8 baseos 36 M 2024-04-10T22:30:37.795 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-04-10T22:30:37.795 INFO:teuthology.orchestra.run.smithi088.stdout:Transaction Summary 2024-04-10T22:30:37.795 INFO:teuthology.orchestra.run.smithi088.stdout:================================================================================ 2024-04-10T22:30:37.795 INFO:teuthology.orchestra.run.smithi088.stdout:Install 3 Packages 2024-04-10T22:30:37.795 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-04-10T22:30:37.796 INFO:teuthology.orchestra.run.smithi088.stdout:Total download size: 90 M 2024-04-10T22:30:37.796 INFO:teuthology.orchestra.run.smithi088.stdout:Installed size: 96 M 2024-04-10T22:30:37.796 INFO:teuthology.orchestra.run.smithi088.stdout:Downloading Packages: 2024-04-10T22:30:39.863 INFO:teuthology.orchestra.run.smithi088.stdout:(1/3): kernel-4.18.0-552.el8.x86_64.rpm 5.5 MB/s | 10 MB 00:01 2024-04-10T22:30:50.041 INFO:teuthology.orchestra.run.smithi088.stdout:(2/3): kernel-modules-4.18.0-552.el8.x86_64.rpm 3.0 MB/s | 36 MB 00:12 2024-04-10T22:30:54.786 INFO:teuthology.orchestra.run.smithi088.stdout:(3/3): kernel-core-4.18.0-552.el8.x86_64.rpm 2.6 MB/s | 44 MB 00:16 2024-04-10T22:30:54.786 INFO:teuthology.orchestra.run.smithi088.stdout:-------------------------------------------------------------------------------- 2024-04-10T22:30:54.786 INFO:teuthology.orchestra.run.smithi088.stdout:Total 5.3 MB/s | 90 MB 00:16 2024-04-10T22:30:55.468 INFO:teuthology.orchestra.run.smithi088.stdout:Running transaction check 2024-04-10T22:30:55.913 INFO:teuthology.orchestra.run.smithi088.stdout:Transaction check succeeded. 2024-04-10T22:30:55.913 INFO:teuthology.orchestra.run.smithi088.stdout:Running transaction test 2024-04-10T22:30:57.170 INFO:teuthology.orchestra.run.smithi088.stdout:Transaction test succeeded. 2024-04-10T22:30:57.202 INFO:teuthology.orchestra.run.smithi088.stdout:Running transaction 2024-04-10T22:31:00.035 INFO:teuthology.orchestra.run.smithi088.stdout: Preparing : 1/1 2024-04-10T22:31:00.641 INFO:teuthology.orchestra.run.smithi088.stdout: Installing : kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:31:04.232 INFO:teuthology.orchestra.run.smithi088.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:31:04.649 INFO:teuthology.orchestra.run.smithi088.stdout: Installing : kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:31:08.086 INFO:teuthology.orchestra.run.smithi088.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:31:08.389 INFO:teuthology.orchestra.run.smithi088.stdout: Installing : kernel-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:31:30.310 INFO:teuthology.orchestra.run.smithi088.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:31:30.335 INFO:teuthology.orchestra.run.smithi088.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:31:30.717 INFO:teuthology.orchestra.run.smithi088.stdout: Running scriptlet: kernel-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:31:30.717 INFO:teuthology.orchestra.run.smithi088.stdout: Verifying : kernel-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:31:30.717 INFO:teuthology.orchestra.run.smithi088.stdout: Verifying : kernel-core-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:31:31.202 INFO:teuthology.orchestra.run.smithi088.stdout: Verifying : kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:31:31.202 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-04-10T22:31:31.202 INFO:teuthology.orchestra.run.smithi088.stdout:Installed: 2024-04-10T22:31:31.202 INFO:teuthology.orchestra.run.smithi088.stdout: kernel-4.18.0-552.el8.x86_64 kernel-core-4.18.0-552.el8.x86_64 2024-04-10T22:31:31.202 INFO:teuthology.orchestra.run.smithi088.stdout: kernel-modules-4.18.0-552.el8.x86_64 2024-04-10T22:31:31.202 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-04-10T22:31:31.202 INFO:teuthology.orchestra.run.smithi088.stdout:Complete! 2024-04-10T22:31:31.354 DEBUG:teuthology.orchestra.run.smithi088:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:31:31.575 INFO:teuthology.orchestra.run.smithi088.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-10T22:31:31.575 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-10T22:31:31.575 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-552.el8.x86_64 2024-04-10T22:31:31.576 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-10T22:31:31.576 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-10T22:31:31.576 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-04-10T22:31:31.576 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi088.front.sepia.ceph.com, path=None, version=distro) 2024-04-10T22:31:31.576 DEBUG:teuthology.orchestra.run.smithi088:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:31:31.786 INFO:teuthology.orchestra.run.smithi088.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-10T22:31:31.787 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-10T22:31:31.787 DEBUG:teuthology.orchestra.run.smithi088:> sudo rpm -qi grub2-tools 2024-04-10T22:31:31.821 INFO:teuthology.orchestra.run.smithi088.stdout:Name : grub2-tools 2024-04-10T22:31:31.821 INFO:teuthology.orchestra.run.smithi088.stdout:Epoch : 1 2024-04-10T22:31:31.821 INFO:teuthology.orchestra.run.smithi088.stdout:Version : 2.02 2024-04-10T22:31:31.821 INFO:teuthology.orchestra.run.smithi088.stdout:Release : 129.el8 2024-04-10T22:31:31.821 INFO:teuthology.orchestra.run.smithi088.stdout:Architecture: x86_64 2024-04-10T22:31:31.821 INFO:teuthology.orchestra.run.smithi088.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2024-04-10T22:31:31.821 INFO:teuthology.orchestra.run.smithi088.stdout:Group : System Environment/Base 2024-04-10T22:31:31.821 INFO:teuthology.orchestra.run.smithi088.stdout:Size : 9265586 2024-04-10T22:31:31.822 INFO:teuthology.orchestra.run.smithi088.stdout:License : GPLv3+ 2024-04-10T22:31:31.822 INFO:teuthology.orchestra.run.smithi088.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2024-04-10T22:31:31.822 INFO:teuthology.orchestra.run.smithi088.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2024-04-10T22:31:31.822 INFO:teuthology.orchestra.run.smithi088.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2024-04-10T22:31:31.822 INFO:teuthology.orchestra.run.smithi088.stdout:Build Host : kbuilder.bsys.centos.org 2024-04-10T22:31:31.822 INFO:teuthology.orchestra.run.smithi088.stdout:Relocations : (not relocatable) 2024-04-10T22:31:31.822 INFO:teuthology.orchestra.run.smithi088.stdout:Packager : CentOS BuildSystem 2024-04-10T22:31:31.822 INFO:teuthology.orchestra.run.smithi088.stdout:Vendor : CentOS 2024-04-10T22:31:31.822 INFO:teuthology.orchestra.run.smithi088.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-10T22:31:31.822 INFO:teuthology.orchestra.run.smithi088.stdout:Summary : Support tools for GRUB. 2024-04-10T22:31:31.822 INFO:teuthology.orchestra.run.smithi088.stdout:Description : 2024-04-10T22:31:31.822 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-04-10T22:31:31.822 INFO:teuthology.orchestra.run.smithi088.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-10T22:31:31.823 INFO:teuthology.orchestra.run.smithi088.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-10T22:31:31.823 INFO:teuthology.orchestra.run.smithi088.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-10T22:31:31.823 INFO:teuthology.orchestra.run.smithi088.stdout:hardware devices. 2024-04-10T22:31:31.823 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-04-10T22:31:31.823 INFO:teuthology.orchestra.run.smithi088.stdout:This subpackage provides tools for support of all platforms. 2024-04-10T22:31:31.824 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-10T22:31:31.825 INFO:teuthology.task.kernel:Updating grub on smithi088 to boot 4.18.0-552.el8.x86_64 2024-04-10T22:31:31.825 DEBUG:teuthology.orchestra.run.smithi088:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-10T22:31:33.645 INFO:teuthology.orchestra.run.smithi088.stderr:Generating grub configuration file ... 2024-04-10T22:31:35.838 INFO:teuthology.orchestra.run.smithi088.stderr:done 2024-04-10T22:31:35.840 DEBUG:teuthology.orchestra.run.smithi088:> mktemp 2024-04-10T22:31:35.872 INFO:teuthology.orchestra.run.smithi088.stdout:/tmp/tmp.TjoayuaEsx 2024-04-10T22:31:35.872 DEBUG:teuthology.orchestra.run.smithi088:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.TjoayuaEsx 2024-04-10T22:31:35.950 DEBUG:teuthology.orchestra.run.smithi088:> sudo chmod 0666 /tmp/tmp.TjoayuaEsx 2024-04-10T22:31:36.090 DEBUG:teuthology.orchestra.remote:smithi088:/tmp/tmp.TjoayuaEsx is 6KB 2024-04-10T22:31:36.138 DEBUG:teuthology.orchestra.run.smithi088:> rm -fr /tmp/tmp.TjoayuaEsx 2024-04-10T22:31:36.156 DEBUG:teuthology.orchestra.run.smithi088:> sudo /bin/ls /boot/loader/entries || true 2024-04-10T22:31:36.261 INFO:teuthology.orchestra.run.smithi088.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2024-04-10T22:31:36.261 INFO:teuthology.orchestra.run.smithi088.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2024-04-10T22:31:36.261 INFO:teuthology.orchestra.run.smithi088.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2024-04-10T22:31:36.261 INFO:teuthology.orchestra.run.smithi088.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64.conf 2024-04-10T22:31:36.262 DEBUG:teuthology.orchestra.run.smithi088:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64 2024-04-10T22:31:36.358 DEBUG:teuthology.orchestra.run.smithi088:> sudo shutdown -r now 2024-04-10T22:32:06.363 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-04-10T22:32:06.364 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi088.front.sepia.ceph.com' 2024-04-10T22:32:06.364 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi088.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:32:24.862 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.88 2024-04-10T22:32:33.863 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi088.front.sepia.ceph.com' 2024-04-10T22:32:33.864 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi088.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:32:36.926 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.88 2024-04-10T22:32:48.927 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi088.front.sepia.ceph.com' 2024-04-10T22:32:48.928 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi088.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:32:51.998 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.88 2024-04-10T22:33:06.999 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi088.front.sepia.ceph.com' 2024-04-10T22:33:07.000 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi088.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:33:25.538 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.88 2024-04-10T22:33:43.539 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi088.front.sepia.ceph.com' 2024-04-10T22:33:43.541 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi088.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:33:44.000 DEBUG:teuthology.orchestra.run.smithi088:> true 2024-04-10T22:33:44.294 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi088.front.sepia.ceph.com' 2024-04-10T22:33:44.294 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "4.18.0-552.el8.x86_64"... 2024-04-10T22:33:44.295 DEBUG:teuthology.orchestra.run.smithi088:> uname -r 2024-04-10T22:33:44.355 INFO:teuthology.orchestra.run.smithi088.stdout:4.18.0-552.el8.x86_64 2024-04-10T22:33:44.356 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-552.el8.x86_64 vs 4.18.0-552.el8.x86_64 2024-04-10T22:33:44.356 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-10T22:33:44.356 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-10T22:33:45.357 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-04-10T22:33:45.357 DEBUG:teuthology.orchestra.run.smithi088:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-10T22:33:45.424 INFO:teuthology.orchestra.run.smithi088.stdout:ttyS1 2024-04-10T22:33:45.439 DEBUG:teuthology.parallel:result is None 2024-04-10T22:34:06.882 INFO:teuthology.orchestra.run.smithi132.stdout:Extra Packages for Enterprise Linux 76 kB/s | 16 MB 03:38 2024-04-10T22:34:10.632 INFO:teuthology.orchestra.run.smithi132.stdout:lab-extras 395 kB/s | 24 kB 00:00 2024-04-10T22:34:10.774 INFO:teuthology.orchestra.run.smithi132.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:34:13.587 INFO:teuthology.orchestra.run.smithi132.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2024-04-10T22:34:13.588 INFO:teuthology.orchestra.run.smithi132.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2024-04-10T22:34:13.699 INFO:teuthology.orchestra.run.smithi132.stdout:Dependencies resolved. 2024-04-10T22:34:13.699 INFO:teuthology.orchestra.run.smithi132.stdout:================================================================================ 2024-04-10T22:34:13.700 INFO:teuthology.orchestra.run.smithi132.stdout: Package Architecture Version Repository Size 2024-04-10T22:34:13.700 INFO:teuthology.orchestra.run.smithi132.stdout:================================================================================ 2024-04-10T22:34:13.700 INFO:teuthology.orchestra.run.smithi132.stdout:Installing: 2024-04-10T22:34:13.700 INFO:teuthology.orchestra.run.smithi132.stdout: kernel x86_64 4.18.0-552.el8 baseos 10 M 2024-04-10T22:34:13.700 INFO:teuthology.orchestra.run.smithi132.stdout:Installing dependencies: 2024-04-10T22:34:13.700 INFO:teuthology.orchestra.run.smithi132.stdout: kernel-core x86_64 4.18.0-552.el8 baseos 44 M 2024-04-10T22:34:13.700 INFO:teuthology.orchestra.run.smithi132.stdout: kernel-modules x86_64 4.18.0-552.el8 baseos 36 M 2024-04-10T22:34:13.700 INFO:teuthology.orchestra.run.smithi132.stdout: 2024-04-10T22:34:13.700 INFO:teuthology.orchestra.run.smithi132.stdout:Transaction Summary 2024-04-10T22:34:13.700 INFO:teuthology.orchestra.run.smithi132.stdout:================================================================================ 2024-04-10T22:34:13.700 INFO:teuthology.orchestra.run.smithi132.stdout:Install 3 Packages 2024-04-10T22:34:13.700 INFO:teuthology.orchestra.run.smithi132.stdout: 2024-04-10T22:34:13.701 INFO:teuthology.orchestra.run.smithi132.stdout:Total download size: 90 M 2024-04-10T22:34:13.701 INFO:teuthology.orchestra.run.smithi132.stdout:Installed size: 96 M 2024-04-10T22:34:13.701 INFO:teuthology.orchestra.run.smithi132.stdout:Downloading Packages: 2024-04-10T22:36:28.534 INFO:teuthology.orchestra.run.smithi132.stdout:(1/3): kernel-4.18.0-552.el8.x86_64.rpm 3.6 MB/s | 10 MB 00:02 2024-04-10T22:36:32.441 INFO:teuthology.orchestra.run.smithi132.stdout:(2/3): kernel-modules-4.18.0-552.el8.x86_64.rpm 5.4 MB/s | 36 MB 00:06 2024-04-10T22:36:40.582 INFO:teuthology.orchestra.run.smithi132.stdout:(3/3): kernel-core-4.18.0-552.el8.x86_64.rpm 2.9 MB/s | 44 MB 00:14 2024-04-10T22:36:40.582 INFO:teuthology.orchestra.run.smithi132.stdout:-------------------------------------------------------------------------------- 2024-04-10T22:36:40.583 INFO:teuthology.orchestra.run.smithi132.stdout:Total 631 kB/s | 90 MB 02:26 2024-04-10T22:36:41.265 INFO:teuthology.orchestra.run.smithi132.stdout:Running transaction check 2024-04-10T22:36:41.712 INFO:teuthology.orchestra.run.smithi132.stdout:Transaction check succeeded. 2024-04-10T22:36:41.712 INFO:teuthology.orchestra.run.smithi132.stdout:Running transaction test 2024-04-10T22:36:42.976 INFO:teuthology.orchestra.run.smithi132.stdout:Transaction test succeeded. 2024-04-10T22:36:43.005 INFO:teuthology.orchestra.run.smithi132.stdout:Running transaction 2024-04-10T22:36:45.836 INFO:teuthology.orchestra.run.smithi132.stdout: Preparing : 1/1 2024-04-10T22:36:46.524 INFO:teuthology.orchestra.run.smithi132.stdout: Installing : kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:36:50.104 INFO:teuthology.orchestra.run.smithi132.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:36:50.515 INFO:teuthology.orchestra.run.smithi132.stdout: Installing : kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:36:53.946 INFO:teuthology.orchestra.run.smithi132.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:36:54.264 INFO:teuthology.orchestra.run.smithi132.stdout: Installing : kernel-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:37:16.234 INFO:teuthology.orchestra.run.smithi132.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:37:16.258 INFO:teuthology.orchestra.run.smithi132.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:37:16.614 INFO:teuthology.orchestra.run.smithi132.stdout: Running scriptlet: kernel-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:37:16.614 INFO:teuthology.orchestra.run.smithi132.stdout: Verifying : kernel-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:37:16.614 INFO:teuthology.orchestra.run.smithi132.stdout: Verifying : kernel-core-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:37:17.703 INFO:teuthology.orchestra.run.smithi132.stdout: Verifying : kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:37:17.703 INFO:teuthology.orchestra.run.smithi132.stdout: 2024-04-10T22:37:17.703 INFO:teuthology.orchestra.run.smithi132.stdout:Installed: 2024-04-10T22:37:17.703 INFO:teuthology.orchestra.run.smithi132.stdout: kernel-4.18.0-552.el8.x86_64 kernel-core-4.18.0-552.el8.x86_64 2024-04-10T22:37:17.703 INFO:teuthology.orchestra.run.smithi132.stdout: kernel-modules-4.18.0-552.el8.x86_64 2024-04-10T22:37:17.703 INFO:teuthology.orchestra.run.smithi132.stdout: 2024-04-10T22:37:17.703 INFO:teuthology.orchestra.run.smithi132.stdout:Complete! 2024-04-10T22:37:17.849 DEBUG:teuthology.orchestra.run.smithi132:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:37:18.065 INFO:teuthology.orchestra.run.smithi132.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-10T22:37:18.065 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-10T22:37:18.065 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-552.el8.x86_64 2024-04-10T22:37:18.066 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-10T22:37:18.066 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-10T22:37:18.066 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-04-10T22:37:18.066 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi132.front.sepia.ceph.com, path=None, version=distro) 2024-04-10T22:37:18.066 DEBUG:teuthology.orchestra.run.smithi132:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:37:18.278 INFO:teuthology.orchestra.run.smithi132.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-10T22:37:18.278 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-10T22:37:18.279 DEBUG:teuthology.orchestra.run.smithi132:> sudo rpm -qi grub2-tools 2024-04-10T22:37:18.312 INFO:teuthology.orchestra.run.smithi132.stdout:Name : grub2-tools 2024-04-10T22:37:18.312 INFO:teuthology.orchestra.run.smithi132.stdout:Epoch : 1 2024-04-10T22:37:18.312 INFO:teuthology.orchestra.run.smithi132.stdout:Version : 2.02 2024-04-10T22:37:18.312 INFO:teuthology.orchestra.run.smithi132.stdout:Release : 129.el8 2024-04-10T22:37:18.312 INFO:teuthology.orchestra.run.smithi132.stdout:Architecture: x86_64 2024-04-10T22:37:18.312 INFO:teuthology.orchestra.run.smithi132.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2024-04-10T22:37:18.312 INFO:teuthology.orchestra.run.smithi132.stdout:Group : System Environment/Base 2024-04-10T22:37:18.312 INFO:teuthology.orchestra.run.smithi132.stdout:Size : 9265586 2024-04-10T22:37:18.312 INFO:teuthology.orchestra.run.smithi132.stdout:License : GPLv3+ 2024-04-10T22:37:18.312 INFO:teuthology.orchestra.run.smithi132.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2024-04-10T22:37:18.312 INFO:teuthology.orchestra.run.smithi132.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2024-04-10T22:37:18.312 INFO:teuthology.orchestra.run.smithi132.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2024-04-10T22:37:18.313 INFO:teuthology.orchestra.run.smithi132.stdout:Build Host : kbuilder.bsys.centos.org 2024-04-10T22:37:18.313 INFO:teuthology.orchestra.run.smithi132.stdout:Relocations : (not relocatable) 2024-04-10T22:37:18.313 INFO:teuthology.orchestra.run.smithi132.stdout:Packager : CentOS BuildSystem 2024-04-10T22:37:18.313 INFO:teuthology.orchestra.run.smithi132.stdout:Vendor : CentOS 2024-04-10T22:37:18.313 INFO:teuthology.orchestra.run.smithi132.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-10T22:37:18.313 INFO:teuthology.orchestra.run.smithi132.stdout:Summary : Support tools for GRUB. 2024-04-10T22:37:18.313 INFO:teuthology.orchestra.run.smithi132.stdout:Description : 2024-04-10T22:37:18.313 INFO:teuthology.orchestra.run.smithi132.stdout: 2024-04-10T22:37:18.313 INFO:teuthology.orchestra.run.smithi132.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-10T22:37:18.313 INFO:teuthology.orchestra.run.smithi132.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-10T22:37:18.313 INFO:teuthology.orchestra.run.smithi132.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-10T22:37:18.313 INFO:teuthology.orchestra.run.smithi132.stdout:hardware devices. 2024-04-10T22:37:18.313 INFO:teuthology.orchestra.run.smithi132.stdout: 2024-04-10T22:37:18.313 INFO:teuthology.orchestra.run.smithi132.stdout:This subpackage provides tools for support of all platforms. 2024-04-10T22:37:18.314 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-10T22:37:18.314 INFO:teuthology.task.kernel:Updating grub on smithi132 to boot 4.18.0-552.el8.x86_64 2024-04-10T22:37:18.314 DEBUG:teuthology.orchestra.run.smithi132:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-10T22:37:19.717 INFO:teuthology.orchestra.run.smithi132.stderr:Generating grub configuration file ... 2024-04-10T22:37:21.580 INFO:teuthology.orchestra.run.smithi132.stderr:done 2024-04-10T22:37:21.582 DEBUG:teuthology.orchestra.run.smithi132:> mktemp 2024-04-10T22:37:21.604 INFO:teuthology.orchestra.run.smithi132.stdout:/tmp/tmp.Kr6iBLpIU8 2024-04-10T22:37:21.605 DEBUG:teuthology.orchestra.run.smithi132:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.Kr6iBLpIU8 2024-04-10T22:37:21.678 DEBUG:teuthology.orchestra.run.smithi132:> sudo chmod 0666 /tmp/tmp.Kr6iBLpIU8 2024-04-10T22:37:21.842 DEBUG:teuthology.orchestra.remote:smithi132:/tmp/tmp.Kr6iBLpIU8 is 6KB 2024-04-10T22:37:21.890 DEBUG:teuthology.orchestra.run.smithi132:> rm -fr /tmp/tmp.Kr6iBLpIU8 2024-04-10T22:37:21.908 DEBUG:teuthology.orchestra.run.smithi132:> sudo /bin/ls /boot/loader/entries || true 2024-04-10T22:37:21.984 INFO:teuthology.orchestra.run.smithi132.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2024-04-10T22:37:21.984 INFO:teuthology.orchestra.run.smithi132.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2024-04-10T22:37:21.984 INFO:teuthology.orchestra.run.smithi132.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2024-04-10T22:37:21.985 INFO:teuthology.orchestra.run.smithi132.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64.conf 2024-04-10T22:37:21.986 DEBUG:teuthology.orchestra.run.smithi132:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64 2024-04-10T22:37:22.198 DEBUG:teuthology.orchestra.run.smithi132:> sudo shutdown -r now 2024-04-10T22:37:52.202 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-04-10T22:37:52.202 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi132.front.sepia.ceph.com' 2024-04-10T22:37:52.203 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi132.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:38:10.718 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.132 2024-04-10T22:38:19.719 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi132.front.sepia.ceph.com' 2024-04-10T22:38:19.720 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi132.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:38:38.110 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.132 2024-04-10T22:38:50.111 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi132.front.sepia.ceph.com' 2024-04-10T22:38:50.112 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi132.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:38:53.182 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.132 2024-04-10T22:39:08.184 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi132.front.sepia.ceph.com' 2024-04-10T22:39:08.184 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi132.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:39:08.496 DEBUG:teuthology.orchestra.run.smithi132:> true 2024-04-10T22:39:08.774 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi132.front.sepia.ceph.com' 2024-04-10T22:39:08.775 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "4.18.0-552.el8.x86_64"... 2024-04-10T22:39:08.775 DEBUG:teuthology.orchestra.run.smithi132:> uname -r 2024-04-10T22:39:08.793 INFO:teuthology.orchestra.run.smithi132.stdout:4.18.0-552.el8.x86_64 2024-04-10T22:39:08.793 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-552.el8.x86_64 vs 4.18.0-552.el8.x86_64 2024-04-10T22:39:08.793 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-10T22:39:08.793 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-10T22:39:09.794 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-04-10T22:39:09.795 DEBUG:teuthology.orchestra.run.smithi132:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-10T22:39:09.889 INFO:teuthology.orchestra.run.smithi132.stdout:ttyS1 2024-04-10T22:39:09.901 DEBUG:teuthology.parallel:result is None 2024-04-10T22:39:09.902 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-10T22:39:09.911 INFO:teuthology.task.internal:Creating test directory... 2024-04-10T22:39:09.911 DEBUG:teuthology.orchestra.run.smithi088:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-10T22:39:09.914 DEBUG:teuthology.orchestra.run.smithi132:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-10T22:39:09.935 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-10T22:39:09.941 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-10T22:39:09.950 INFO:teuthology.task.internal:Creating archive directory... 2024-04-10T22:39:09.950 DEBUG:teuthology.orchestra.run.smithi088:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-10T22:39:09.976 DEBUG:teuthology.orchestra.run.smithi132:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-10T22:39:10.017 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-10T22:39:10.024 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-10T22:39:10.024 DEBUG:teuthology.orchestra.run.smithi088:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-10T22:39:10.055 DEBUG:teuthology.orchestra.run.smithi132:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-10T22:39:10.090 INFO:teuthology.orchestra.run.smithi132.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:39:10.097 INFO:teuthology.orchestra.run.smithi088.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:39:10.101 INFO:teuthology.orchestra.run.smithi132.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:39:10.108 INFO:teuthology.orchestra.run.smithi088.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:39:10.110 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-10T22:39:10.117 INFO:teuthology.task.internal:Configuring sudo... 2024-04-10T22:39:10.117 DEBUG:teuthology.orchestra.run.smithi088:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-10T22:39:10.119 DEBUG:teuthology.orchestra.run.smithi132:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-10T22:39:10.171 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-10T22:39:10.180 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-10T22:39:10.181 DEBUG:teuthology.orchestra.run.smithi088:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-10T22:39:10.189 DEBUG:teuthology.orchestra.run.smithi132:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-10T22:39:10.231 DEBUG:teuthology.orchestra.run.smithi088:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:39:10.289 DEBUG:teuthology.orchestra.run.smithi088:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:39:10.351 DEBUG:teuthology.orchestra.run.smithi088:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:39:10.389 DEBUG:teuthology.orchestra.run.smithi088:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:39:10.438 DEBUG:teuthology.orchestra.run.smithi088:> set -ex 2024-04-10T22:39:10.438 DEBUG:teuthology.orchestra.run.smithi088:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-10T22:39:10.504 DEBUG:teuthology.orchestra.run.smithi132:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:39:10.550 DEBUG:teuthology.orchestra.run.smithi132:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:39:10.612 DEBUG:teuthology.orchestra.run.smithi132:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:39:10.658 DEBUG:teuthology.orchestra.run.smithi132:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:39:10.701 DEBUG:teuthology.orchestra.run.smithi132:> set -ex 2024-04-10T22:39:10.702 DEBUG:teuthology.orchestra.run.smithi132:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-10T22:39:10.770 DEBUG:teuthology.orchestra.run.smithi088:> sudo service rsyslog restart 2024-04-10T22:39:10.772 DEBUG:teuthology.orchestra.run.smithi132:> sudo service rsyslog restart 2024-04-10T22:39:10.851 INFO:teuthology.orchestra.run.smithi088.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:39:10.877 INFO:teuthology.orchestra.run.smithi132.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:39:11.131 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-10T22:39:11.140 INFO:teuthology.task.internal:Starting timer... 2024-04-10T22:39:11.140 INFO:teuthology.run_tasks:Running task pcp... 2024-04-10T22:39:11.152 INFO:teuthology.run_tasks:Running task selinux... 2024-04-10T22:39:11.160 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-04-10T22:39:11.160 DEBUG:teuthology.orchestra.run.smithi088:> sudo service auditd rotate 2024-04-10T22:39:11.236 INFO:teuthology.orchestra.run.smithi088.stdout:Rotating logs: 2024-04-10T22:39:11.238 DEBUG:teuthology.orchestra.run.smithi132:> sudo service auditd rotate 2024-04-10T22:39:11.313 INFO:teuthology.orchestra.run.smithi132.stdout:Rotating logs: 2024-04-10T22:39:11.314 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-10T22:39:11.315 DEBUG:teuthology.orchestra.run.smithi088:> /usr/sbin/getenforce 2024-04-10T22:39:11.352 INFO:teuthology.orchestra.run.smithi088.stdout:Permissive 2024-04-10T22:39:11.352 DEBUG:teuthology.orchestra.run.smithi132:> /usr/sbin/getenforce 2024-04-10T22:39:11.389 INFO:teuthology.orchestra.run.smithi132.stdout:Permissive 2024-04-10T22:39:11.389 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi088.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi132.front.sepia.ceph.com': 'permissive'} 2024-04-10T22:39:11.390 DEBUG:teuthology.orchestra.run.smithi088:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:39:11.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:39:11.421 DEBUG:teuthology.orchestra.run.smithi132:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:39:11.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:39:11.457 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-10T22:39:11.457 DEBUG:teuthology.orchestra.run.smithi088:> sudo /usr/sbin/setenforce permissive 2024-04-10T22:39:11.488 DEBUG:teuthology.orchestra.run.smithi132:> sudo /usr/sbin/setenforce permissive 2024-04-10T22:39:11.525 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-10T22:39:11.535 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-04-10T22:39:11.540 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-04-10T22:39:11.674 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-10T22:39:11.689 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-10T22:39:11.691 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi088.front.sepia.ceph.com,smithi132.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-10T22:51:20.593 INFO:teuthology.task.ansible:Archiving ansible failure log at: /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650169/ansible_failures.yaml 2024-04-10T22:51:20.595 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/__init__.py", line 123, in __enter__ self.begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 508, in begin super(CephLab, self).begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 342, in begin self.execute_playbook() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 374, in execute_playbook self._handle_failure(command, status) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 402, in _handle_failure raise AnsibleFailedError(failures) teuthology.exceptions.AnsibleFailedError: [',', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at'] 2024-04-10T22:51:20.820 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=74f1fbb350e14aadb4bf97e2431f2bb4 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/__init__.py", line 123, in __enter__ self.begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 508, in begin super(CephLab, self).begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 342, in begin self.execute_playbook() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 374, in execute_playbook self._handle_failure(command, status) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 402, in _handle_failure raise AnsibleFailedError(failures) teuthology.exceptions.AnsibleFailedError: [',', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at'] 2024-04-10T22:51:20.823 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-10T22:51:20.831 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-10T22:51:20.831 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-10T22:51:20.839 DEBUG:teuthology.orchestra.run.smithi088:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-10T22:51:20.907 DEBUG:teuthology.orchestra.run.smithi132:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-10T22:51:20.972 DEBUG:teuthology.orchestra.run.smithi088:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:51:21.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:51:21.043 DEBUG:teuthology.orchestra.run.smithi132:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:51:21.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:51:21.075 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-10T22:51:21.089 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-10T22:51:21.096 INFO:teuthology.task.internal:Duration was 729.956755 seconds 2024-04-10T22:51:21.097 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-10T22:51:21.104 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-10T22:51:21.104 DEBUG:teuthology.orchestra.run.smithi088:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-10T22:51:21.106 DEBUG:teuthology.orchestra.run.smithi132:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-10T22:51:21.153 INFO:teuthology.orchestra.run.smithi088.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:51:21.161 INFO:teuthology.orchestra.run.smithi132.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:51:21.630 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-10T22:51:21.631 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi088.front.sepia.ceph.com 2024-04-10T22:51:21.631 DEBUG:teuthology.orchestra.run.smithi088:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-10T22:51:21.655 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi132.front.sepia.ceph.com 2024-04-10T22:51:21.655 DEBUG:teuthology.orchestra.run.smithi132:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-10T22:51:21.677 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-10T22:51:21.678 DEBUG:teuthology.orchestra.run.smithi088:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-10T22:51:21.698 DEBUG:teuthology.orchestra.run.smithi132:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-10T22:51:21.758 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-10T22:51:21.758 DEBUG:teuthology.orchestra.run.smithi088:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-10T22:51:21.779 DEBUG:teuthology.orchestra.run.smithi132:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-10T22:51:21.859 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-10T22:51:21.868 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-10T22:51:21.868 DEBUG:teuthology.orchestra.run.smithi088:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-10T22:51:21.895 DEBUG:teuthology.orchestra.run.smithi132:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-10T22:51:21.921 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-10T22:51:21.930 DEBUG:teuthology.orchestra.run.smithi088:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-10T22:51:21.937 DEBUG:teuthology.orchestra.run.smithi132:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-10T22:51:21.962 INFO:teuthology.orchestra.run.smithi088.stdout:kernel.core_pattern = core 2024-04-10T22:51:21.988 INFO:teuthology.orchestra.run.smithi132.stdout:kernel.core_pattern = core 2024-04-10T22:51:22.008 DEBUG:teuthology.orchestra.run.smithi088:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-10T22:51:22.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:51:22.038 DEBUG:teuthology.orchestra.run.smithi132:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-10T22:51:22.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:51:22.066 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-10T22:51:22.075 INFO:teuthology.task.internal:Transferring archived files... 2024-04-10T22:51:22.076 DEBUG:teuthology.misc:Transferring archived files from smithi088:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650169/remote/smithi088 2024-04-10T22:51:22.077 DEBUG:teuthology.orchestra.run.smithi088:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-10T22:51:22.121 DEBUG:teuthology.misc:Transferring archived files from smithi132:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650169/remote/smithi132 2024-04-10T22:51:22.122 DEBUG:teuthology.orchestra.run.smithi132:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-10T22:51:22.172 INFO:teuthology.task.internal:Removing archive directory... 2024-04-10T22:51:22.172 DEBUG:teuthology.orchestra.run.smithi088:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-10T22:51:22.175 DEBUG:teuthology.orchestra.run.smithi132:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-10T22:51:22.224 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-10T22:51:22.235 INFO:teuthology.task.internal:Not uploading archives. 2024-04-10T22:51:22.235 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-10T22:51:22.243 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-10T22:51:22.243 DEBUG:teuthology.orchestra.run.smithi088:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-10T22:51:22.245 DEBUG:teuthology.orchestra.run.smithi132:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-10T22:51:22.260 INFO:teuthology.orchestra.run.smithi088.stdout: 393224 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 10 22:51 /home/ubuntu/cephtest 2024-04-10T22:51:22.282 INFO:teuthology.orchestra.run.smithi132.stdout: 525362 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 10 22:51 /home/ubuntu/cephtest 2024-04-10T22:51:22.283 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-10T22:51:22.292 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-10T22:51:22.318 INFO:teuthology.nuke:Checking targets against current locks 2024-04-10T22:51:22.345 DEBUG:teuthology.nuke:shortname: smithi088 2024-04-10T22:51:22.346 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T22:51:22.367 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi088.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650169', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:15:48.871204', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBIHgHKExPoXrhhBorDqIfE8KUwKOaIzeUXMjkWH25sbPe4rcItwywAWJhGVhgRocA8ezsdJwTfhX7Zb5BuxqImQ='} 2024-04-10T22:51:22.373 DEBUG:teuthology.nuke:shortname: smithi132 2024-04-10T22:51:22.374 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T22:51:22.396 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi132.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650169', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:15:48.872821', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBG4qGLVgVzTPUpDraSjMUYExRwLyaPdU2mp2Ux6Qq6hJogoS7kC5Keg4tuThvW2GuEvgqvfNp5u91EuAAcLoH4I='} 2024-04-10T22:51:22.420 INFO:teuthology.orchestra.console.smithi088:Power off 2024-04-10T22:51:22.420 DEBUG:teuthology.orchestra.console.smithi088:pexpect command: ipmitool -H smithi088.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-10T22:51:22.444 DEBUG:teuthology.orchestra.console.smithi088:power off output: Chassis Power Control: Down/Off 2024-04-10T22:51:22.445 DEBUG:teuthology.orchestra.console.smithi088:pexpect command: ipmitool -H smithi088.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:51:22.461 INFO:teuthology.orchestra.console.smithi132:Power off 2024-04-10T22:51:22.461 DEBUG:teuthology.orchestra.console.smithi132:pexpect command: ipmitool -H smithi132.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-10T22:51:22.471 DEBUG:teuthology.orchestra.console.smithi088:check power output: Chassis Power is on 2024-04-10T22:51:22.484 DEBUG:teuthology.orchestra.console.smithi132:power off output: Chassis Power Control: Down/Off 2024-04-10T22:51:22.484 DEBUG:teuthology.orchestra.console.smithi132:pexpect command: ipmitool -H smithi132.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:51:22.513 DEBUG:teuthology.orchestra.console.smithi132:check power output: Chassis Power is on 2024-04-10T22:51:26.473 DEBUG:teuthology.orchestra.console.smithi088:pexpect command: ipmitool -H smithi088.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:51:26.515 DEBUG:teuthology.orchestra.console.smithi132:pexpect command: ipmitool -H smithi132.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:51:26.583 DEBUG:teuthology.orchestra.console.smithi088:check power output: Chassis Power is on 2024-04-10T22:51:26.625 DEBUG:teuthology.orchestra.console.smithi132:check power output: Chassis Power is on 2024-04-10T22:51:30.589 DEBUG:teuthology.orchestra.console.smithi088:pexpect command: ipmitool -H smithi088.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:51:30.626 DEBUG:teuthology.orchestra.console.smithi132:pexpect command: ipmitool -H smithi132.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:51:30.699 DEBUG:teuthology.orchestra.console.smithi088:check power output: Chassis Power is on 2024-04-10T22:51:30.735 DEBUG:teuthology.orchestra.console.smithi132:check power output: Chassis Power is on 2024-04-10T22:51:34.702 DEBUG:teuthology.orchestra.console.smithi088:pexpect command: ipmitool -H smithi088.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:51:34.738 DEBUG:teuthology.orchestra.console.smithi132:pexpect command: ipmitool -H smithi132.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:51:34.812 DEBUG:teuthology.orchestra.console.smithi088:check power output: Chassis Power is off 2024-04-10T22:51:34.847 DEBUG:teuthology.orchestra.console.smithi132:check power output: Chassis Power is off 2024-04-10T22:51:34.914 INFO:teuthology.orchestra.console.smithi088:Power off completed 2024-04-10T22:51:34.949 INFO:teuthology.orchestra.console.smithi132:Power off completed 2024-04-10T22:51:35.055 INFO:teuthology.run:Summary data: description: upgrade:pacific-x/stress-split/{0-distro/centos_8.stream_container_tools_crun 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} duration: 729.956755399704 failure_reason: '['','', ''Bugzilla. In case of problems, contact the owner of this repository.'', ''Enabling a Copr repository. Please note that this repository is not part'', ''Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable'', ''Please do not file bug reports about these packages in Fedora'', ''The Fedora Project does not exercise any power over the contents of'', ''and packages are not held to any quality or security level.'', ''of the main distribution, and quality may vary.'', ''this repository beyond the rules outlined in the Copr FAQ at'']' owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=74f1fbb350e14aadb4bf97e2431f2bb4 status: dead success: false 2024-04-10T22:51:35.055 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-10T22:51:35.145 INFO:teuthology.run:DEAD