2024-04-10T22:37:58.015 INFO:root:teuthology version: 0.0.1.dev267+g6c63784 2024-04-10T22:37:58.015 DEBUG:teuthology.run:Teuthology command: teuthology --verbose --archive /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650179 --description upgrade:pacific-x/stress-split/{0-distro/centos_8.stream_container_tools_crun 0-roles 1-start 2-first-half-tasks/snaps-few-objects 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} --owner scheduled_teuthology@teuthology --name teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi -- /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650179/orig.config.yaml 2024-04-10T22:37:58.075 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-10T22:37:58.173 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650179 branch: reef description: upgrade:pacific-x/stress-split/{0-distro/centos_8.stream_container_tools_crun 0-roles 1-start 2-first-half-tasks/snaps-few-objects 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} email: ceph-qa@ceph.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' first_in_suite: false job_id: '7650179' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run basic import/export cli tests for rbd ' name: teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 8.stream overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: reef sha1: 4f70a9a90881107996c8b7a3e89ba8b32624713c owner: scheduled_teuthology@teuthology priority: 830 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' seed: 4604 sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: pacific clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 0/1 suite: upgrade:pacific-x suite_branch: reef suite_path: /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 4f70a9a90881107996c8b7a3e89ba8b32624713c targets: smithi040.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLZ4p8PrKjn8Hebat7IWFTE+zKMwC2ccn2QbL10klKSaZqlrtnQn1GTdqW2/KJgD+Zp7kNXwEmkrNk+rQBTekPA= smithi098.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGZSZ0+yeaC3Dd+nqKVCgswtB7OhcXx+IPRijqlWeQAfHpmKnTWmoyifLK7J/WEniFf3eOCubEXwBHY4lYJe2us= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - sudo sed -i 's/runtime = "runc"/#runtime = "runc"/g' /usr/share/containers/containers.conf - sudo sed -i 's/#runtime = "crun"/runtime = "crun"/g' /usr/share/containers/containers.conf - install: branch: pacific exclude_packages: - ceph-volume - cephadm: cephadm_branch: pacific cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:pacific - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 6c637841c215537a4502385240412f1966e0faab timestamp: 2024-04-10_01:08:08 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-04-10T22:37:58.174 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa; will attempt to use it 2024-04-10T22:37:58.174 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa/tasks 2024-04-10T22:37:58.175 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-10T22:37:58.177 INFO:teuthology.task.internal:Checking packages... 2024-04-10T22:37:58.199 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'd540ebaca6b131a1dd560e7f69e024b133bbaa42' 2024-04-10T22:37:58.199 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-10T22:37:58.199 INFO:teuthology.packaging:ref: None 2024-04-10T22:37:58.199 INFO:teuthology.packaging:tag: None 2024-04-10T22:37:58.199 INFO:teuthology.packaging:branch: reef 2024-04-10T22:37:58.199 INFO:teuthology.packaging:sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 2024-04-10T22:37:58.199 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef 2024-04-10T22:37:58.547 INFO:teuthology.task.internal:Found packages for ceph version 18.2.2-1189.gc96c8d47 2024-04-10T22:37:58.548 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-10T22:37:58.556 INFO:teuthology.task.internal:no buildpackages task found 2024-04-10T22:37:58.556 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-10T22:37:58.562 INFO:teuthology.task.internal:Saving configuration 2024-04-10T22:37:58.585 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-10T22:37:58.590 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T22:37:58.608 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi040.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650179', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:15.067322', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLZ4p8PrKjn8Hebat7IWFTE+zKMwC2ccn2QbL10klKSaZqlrtnQn1GTdqW2/KJgD+Zp7kNXwEmkrNk+rQBTekPA='} 2024-04-10T22:37:58.626 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi098.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650179', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:15.068846', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGZSZ0+yeaC3Dd+nqKVCgswtB7OhcXx+IPRijqlWeQAfHpmKnTWmoyifLK7J/WEniFf3eOCubEXwBHY4lYJe2us='} 2024-04-10T22:37:58.626 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-10T22:37:58.655 INFO:teuthology.task.internal:roles: ubuntu@smithi040.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-04-10T22:37:58.655 INFO:teuthology.task.internal:roles: ubuntu@smithi098.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-04-10T22:37:58.655 INFO:teuthology.run_tasks:Running task console_log... 2024-04-10T22:37:58.715 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f246831a0d0>, signals=[15]) 2024-04-10T22:37:58.715 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-10T22:37:58.721 INFO:teuthology.task.internal:Opening connections... 2024-04-10T22:37:58.721 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi040.front.sepia.ceph.com 2024-04-10T22:37:58.723 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi040.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:37:58.800 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi098.front.sepia.ceph.com 2024-04-10T22:37:58.801 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi098.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:37:58.876 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-10T22:37:58.883 DEBUG:teuthology.orchestra.run.smithi040:> uname -m 2024-04-10T22:37:58.906 INFO:teuthology.orchestra.run.smithi040.stdout:x86_64 2024-04-10T22:37:58.906 DEBUG:teuthology.orchestra.run.smithi040:> cat /etc/os-release 2024-04-10T22:37:58.968 INFO:teuthology.orchestra.run.smithi040.stdout:NAME="CentOS Stream" 2024-04-10T22:37:58.968 INFO:teuthology.orchestra.run.smithi040.stdout:VERSION="8" 2024-04-10T22:37:58.968 INFO:teuthology.orchestra.run.smithi040.stdout:ID="centos" 2024-04-10T22:37:58.968 INFO:teuthology.orchestra.run.smithi040.stdout:ID_LIKE="rhel fedora" 2024-04-10T22:37:58.968 INFO:teuthology.orchestra.run.smithi040.stdout:VERSION_ID="8" 2024-04-10T22:37:58.969 INFO:teuthology.orchestra.run.smithi040.stdout:PLATFORM_ID="platform:el8" 2024-04-10T22:37:58.969 INFO:teuthology.orchestra.run.smithi040.stdout:PRETTY_NAME="CentOS Stream 8" 2024-04-10T22:37:58.969 INFO:teuthology.orchestra.run.smithi040.stdout:ANSI_COLOR="0;31" 2024-04-10T22:37:58.969 INFO:teuthology.orchestra.run.smithi040.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2024-04-10T22:37:58.969 INFO:teuthology.orchestra.run.smithi040.stdout:HOME_URL="https://centos.org/" 2024-04-10T22:37:58.969 INFO:teuthology.orchestra.run.smithi040.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-10T22:37:58.969 INFO:teuthology.orchestra.run.smithi040.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2024-04-10T22:37:58.969 INFO:teuthology.orchestra.run.smithi040.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-10T22:37:58.969 INFO:teuthology.lock.ops:Updating smithi040.front.sepia.ceph.com on lock server 2024-04-10T22:37:58.990 DEBUG:teuthology.orchestra.run.smithi098:> uname -m 2024-04-10T22:37:59.012 INFO:teuthology.orchestra.run.smithi098.stdout:x86_64 2024-04-10T22:37:59.012 DEBUG:teuthology.orchestra.run.smithi098:> cat /etc/os-release 2024-04-10T22:37:59.072 INFO:teuthology.orchestra.run.smithi098.stdout:NAME="CentOS Stream" 2024-04-10T22:37:59.072 INFO:teuthology.orchestra.run.smithi098.stdout:VERSION="8" 2024-04-10T22:37:59.072 INFO:teuthology.orchestra.run.smithi098.stdout:ID="centos" 2024-04-10T22:37:59.072 INFO:teuthology.orchestra.run.smithi098.stdout:ID_LIKE="rhel fedora" 2024-04-10T22:37:59.072 INFO:teuthology.orchestra.run.smithi098.stdout:VERSION_ID="8" 2024-04-10T22:37:59.073 INFO:teuthology.orchestra.run.smithi098.stdout:PLATFORM_ID="platform:el8" 2024-04-10T22:37:59.073 INFO:teuthology.orchestra.run.smithi098.stdout:PRETTY_NAME="CentOS Stream 8" 2024-04-10T22:37:59.073 INFO:teuthology.orchestra.run.smithi098.stdout:ANSI_COLOR="0;31" 2024-04-10T22:37:59.073 INFO:teuthology.orchestra.run.smithi098.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2024-04-10T22:37:59.073 INFO:teuthology.orchestra.run.smithi098.stdout:HOME_URL="https://centos.org/" 2024-04-10T22:37:59.073 INFO:teuthology.orchestra.run.smithi098.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-10T22:37:59.073 INFO:teuthology.orchestra.run.smithi098.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2024-04-10T22:37:59.073 INFO:teuthology.orchestra.run.smithi098.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-10T22:37:59.073 INFO:teuthology.lock.ops:Updating smithi098.front.sepia.ceph.com on lock server 2024-04-10T22:37:59.089 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-10T22:37:59.099 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-10T22:37:59.105 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-10T22:37:59.105 DEBUG:teuthology.orchestra.run.smithi040:> test '!' -e /home/ubuntu/cephtest 2024-04-10T22:37:59.107 DEBUG:teuthology.orchestra.run.smithi098:> test '!' -e /home/ubuntu/cephtest 2024-04-10T22:37:59.132 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-10T22:37:59.138 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-10T22:37:59.138 DEBUG:teuthology.orchestra.run.smithi040:> test -z $(ls -A /var/lib/ceph) 2024-04-10T22:37:59.168 DEBUG:teuthology.orchestra.run.smithi098:> test -z $(ls -A /var/lib/ceph) 2024-04-10T22:37:59.200 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-10T22:37:59.247 INFO:teuthology.run_tasks:Running task kernel... 2024-04-10T22:37:59.261 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-04-10T22:37:59.261 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-04-10T22:37:59.262 DEBUG:teuthology.orchestra.run.smithi040:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-10T22:37:59.262 DEBUG:teuthology.orchestra.run.smithi098:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-10T22:37:59.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:37:59.281 DEBUG:teuthology.orchestra.run.smithi098:> uname -r 2024-04-10T22:37:59.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:37:59.284 DEBUG:teuthology.orchestra.run.smithi040:> uname -r 2024-04-10T22:37:59.339 INFO:teuthology.orchestra.run.smithi098.stdout:4.18.0-489.el8.x86_64 2024-04-10T22:37:59.339 INFO:teuthology.task.kernel:Running kernel on smithi098: 4.18.0-489.el8.x86_64 2024-04-10T22:37:59.339 DEBUG:teuthology.orchestra.run.smithi098:> sudo yum install -y kernel 2024-04-10T22:37:59.345 INFO:teuthology.orchestra.run.smithi040.stdout:4.18.0-489.el8.x86_64 2024-04-10T22:37:59.345 INFO:teuthology.task.kernel:Running kernel on smithi040: 4.18.0-489.el8.x86_64 2024-04-10T22:37:59.345 DEBUG:teuthology.orchestra.run.smithi040:> sudo yum install -y kernel 2024-04-10T22:38:03.492 INFO:teuthology.orchestra.run.smithi040.stdout:CentOS Stream 8 - AppStream 18 MB/s | 28 MB 00:01 2024-04-10T22:38:03.515 INFO:teuthology.orchestra.run.smithi098.stdout:CentOS Stream 8 - AppStream 17 MB/s | 28 MB 00:01 2024-04-10T22:38:09.540 INFO:teuthology.orchestra.run.smithi098.stdout:CentOS Stream 8 - BaseOS 9.8 MB/s | 10 MB 00:01 2024-04-10T22:38:10.792 INFO:teuthology.orchestra.run.smithi040.stdout:CentOS Stream 8 - BaseOS 5.5 MB/s | 10 MB 00:01 2024-04-10T22:38:11.907 INFO:teuthology.orchestra.run.smithi098.stdout:CentOS Stream 8 - Extras 23 kB/s | 18 kB 00:00 2024-04-10T22:38:12.951 INFO:teuthology.orchestra.run.smithi040.stdout:CentOS Stream 8 - Extras 41 kB/s | 18 kB 00:00 2024-04-10T22:38:13.410 INFO:teuthology.orchestra.run.smithi040.stdout:CentOS Stream 8 - Extras common packages 23 kB/s | 7.7 kB 00:00 2024-04-10T22:38:55.003 INFO:teuthology.orchestra.run.smithi040.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:41 2024-04-10T22:38:55.003 INFO:teuthology.orchestra.run.smithi040.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:38:55.003 INFO:teuthology.orchestra.run.smithi040.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:38:55.004 INFO:teuthology.orchestra.run.smithi040.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:40:21.161 INFO:teuthology.orchestra.run.smithi098.stdout:CentOS Stream 8 - Extras common packages 61 B/s | 7.7 kB 02:09 2024-04-10T22:41:05.946 INFO:teuthology.orchestra.run.smithi098.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:44 2024-04-10T22:41:05.947 INFO:teuthology.orchestra.run.smithi098.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:41:05.947 INFO:teuthology.orchestra.run.smithi098.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:41:05.947 INFO:teuthology.orchestra.run.smithi098.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:41:15.275 INFO:teuthology.orchestra.run.smithi040.stdout:Extra Packages for Enterprise Linux 119 kB/s | 16 MB 02:20 2024-04-10T22:41:19.249 INFO:teuthology.orchestra.run.smithi040.stdout:lab-extras 438 kB/s | 24 kB 00:00 2024-04-10T22:41:19.406 INFO:teuthology.orchestra.run.smithi040.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:41:21.814 INFO:teuthology.orchestra.run.smithi098.stdout:Extra Packages for Enterprise Linux 1.0 MB/s | 16 MB 00:15 2024-04-10T22:41:22.422 INFO:teuthology.orchestra.run.smithi040.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2024-04-10T22:41:22.422 INFO:teuthology.orchestra.run.smithi040.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2024-04-10T22:41:22.528 INFO:teuthology.orchestra.run.smithi040.stdout:Dependencies resolved. 2024-04-10T22:41:22.529 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2024-04-10T22:41:22.529 INFO:teuthology.orchestra.run.smithi040.stdout: Package Architecture Version Repository Size 2024-04-10T22:41:22.529 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2024-04-10T22:41:22.529 INFO:teuthology.orchestra.run.smithi040.stdout:Installing: 2024-04-10T22:41:22.530 INFO:teuthology.orchestra.run.smithi040.stdout: kernel x86_64 4.18.0-552.el8 baseos 10 M 2024-04-10T22:41:22.530 INFO:teuthology.orchestra.run.smithi040.stdout:Installing dependencies: 2024-04-10T22:41:22.530 INFO:teuthology.orchestra.run.smithi040.stdout: kernel-core x86_64 4.18.0-552.el8 baseos 44 M 2024-04-10T22:41:22.530 INFO:teuthology.orchestra.run.smithi040.stdout: kernel-modules x86_64 4.18.0-552.el8 baseos 36 M 2024-04-10T22:41:22.530 INFO:teuthology.orchestra.run.smithi040.stdout: 2024-04-10T22:41:22.530 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction Summary 2024-04-10T22:41:22.530 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2024-04-10T22:41:22.530 INFO:teuthology.orchestra.run.smithi040.stdout:Install 3 Packages 2024-04-10T22:41:22.530 INFO:teuthology.orchestra.run.smithi040.stdout: 2024-04-10T22:41:22.531 INFO:teuthology.orchestra.run.smithi040.stdout:Total download size: 90 M 2024-04-10T22:41:22.531 INFO:teuthology.orchestra.run.smithi040.stdout:Installed size: 96 M 2024-04-10T22:41:22.531 INFO:teuthology.orchestra.run.smithi040.stdout:Downloading Packages: 2024-04-10T22:41:25.606 INFO:teuthology.orchestra.run.smithi098.stdout:lab-extras 421 kB/s | 24 kB 00:00 2024-04-10T22:41:25.747 INFO:teuthology.orchestra.run.smithi098.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:41:26.709 INFO:teuthology.orchestra.run.smithi040.stdout:(1/3): kernel-4.18.0-552.el8.x86_64.rpm 2.6 MB/s | 10 MB 00:03 2024-04-10T22:41:28.528 INFO:teuthology.orchestra.run.smithi098.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2024-04-10T22:41:28.528 INFO:teuthology.orchestra.run.smithi098.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2024-04-10T22:41:28.646 INFO:teuthology.orchestra.run.smithi098.stdout:Dependencies resolved. 2024-04-10T22:41:28.646 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2024-04-10T22:41:28.647 INFO:teuthology.orchestra.run.smithi098.stdout: Package Architecture Version Repository Size 2024-04-10T22:41:28.647 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2024-04-10T22:41:28.647 INFO:teuthology.orchestra.run.smithi098.stdout:Installing: 2024-04-10T22:41:28.647 INFO:teuthology.orchestra.run.smithi098.stdout: kernel x86_64 4.18.0-552.el8 baseos 10 M 2024-04-10T22:41:28.647 INFO:teuthology.orchestra.run.smithi098.stdout:Installing dependencies: 2024-04-10T22:41:28.647 INFO:teuthology.orchestra.run.smithi098.stdout: kernel-core x86_64 4.18.0-552.el8 baseos 44 M 2024-04-10T22:41:28.647 INFO:teuthology.orchestra.run.smithi098.stdout: kernel-modules x86_64 4.18.0-552.el8 baseos 36 M 2024-04-10T22:41:28.647 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-04-10T22:41:28.647 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction Summary 2024-04-10T22:41:28.647 INFO:teuthology.orchestra.run.smithi098.stdout:================================================================================ 2024-04-10T22:41:28.647 INFO:teuthology.orchestra.run.smithi098.stdout:Install 3 Packages 2024-04-10T22:41:28.647 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-04-10T22:41:28.648 INFO:teuthology.orchestra.run.smithi098.stdout:Total download size: 90 M 2024-04-10T22:41:28.648 INFO:teuthology.orchestra.run.smithi098.stdout:Installed size: 96 M 2024-04-10T22:41:28.648 INFO:teuthology.orchestra.run.smithi098.stdout:Downloading Packages: 2024-04-10T22:41:35.374 INFO:teuthology.orchestra.run.smithi098.stdout:(1/3): kernel-core-4.18.0-552.el8.x86_64.rpm 6.7 MB/s | 44 MB 00:06 2024-04-10T22:41:35.978 INFO:teuthology.orchestra.run.smithi098.stdout:(2/3): kernel-4.18.0-552.el8.x86_64.rpm 1.5 MB/s | 10 MB 00:07 2024-04-10T22:41:44.207 INFO:teuthology.orchestra.run.smithi040.stdout:(2/3): kernel-modules-4.18.0-552.el8.x86_64.rpm 1.7 MB/s | 36 MB 00:21 2024-04-10T22:41:51.640 INFO:teuthology.orchestra.run.smithi040.stdout:(3/3): kernel-core-4.18.0-552.el8.x86_64.rpm 1.5 MB/s | 44 MB 00:28 2024-04-10T22:41:51.640 INFO:teuthology.orchestra.run.smithi040.stdout:-------------------------------------------------------------------------------- 2024-04-10T22:41:51.640 INFO:teuthology.orchestra.run.smithi040.stdout:Total 3.1 MB/s | 90 MB 00:29 2024-04-10T22:41:52.362 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction check 2024-04-10T22:41:52.816 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction check succeeded. 2024-04-10T22:41:52.816 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction test 2024-04-10T22:41:54.140 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction test succeeded. 2024-04-10T22:41:54.176 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction 2024-04-10T22:41:54.531 INFO:teuthology.orchestra.run.smithi098.stdout:(3/3): kernel-modules-4.18.0-552.el8.x86_64.rpm 1.4 MB/s | 36 MB 00:25 2024-04-10T22:41:54.531 INFO:teuthology.orchestra.run.smithi098.stdout:-------------------------------------------------------------------------------- 2024-04-10T22:41:54.531 INFO:teuthology.orchestra.run.smithi098.stdout:Total 3.5 MB/s | 90 MB 00:25 2024-04-10T22:41:55.223 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction check 2024-04-10T22:41:55.653 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction check succeeded. 2024-04-10T22:41:55.653 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction test 2024-04-10T22:41:56.991 INFO:teuthology.orchestra.run.smithi098.stdout:Transaction test succeeded. 2024-04-10T22:41:57.013 INFO:teuthology.orchestra.run.smithi098.stdout:Running transaction 2024-04-10T22:41:57.135 INFO:teuthology.orchestra.run.smithi040.stdout: Preparing : 1/1 2024-04-10T22:41:57.711 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:41:59.783 INFO:teuthology.orchestra.run.smithi098.stdout: Preparing : 1/1 2024-04-10T22:42:00.312 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:42:01.408 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:42:01.802 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:42:03.861 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:42:04.262 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:42:05.484 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:42:05.872 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : kernel-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:42:07.705 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:42:08.003 INFO:teuthology.orchestra.run.smithi098.stdout: Installing : kernel-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:42:29.253 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:42:29.278 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:42:29.654 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: kernel-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:42:29.654 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : kernel-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:42:29.655 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : kernel-core-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:42:29.886 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:42:29.909 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:42:30.153 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:42:30.153 INFO:teuthology.orchestra.run.smithi040.stdout: 2024-04-10T22:42:30.153 INFO:teuthology.orchestra.run.smithi040.stdout:Installed: 2024-04-10T22:42:30.153 INFO:teuthology.orchestra.run.smithi040.stdout: kernel-4.18.0-552.el8.x86_64 kernel-core-4.18.0-552.el8.x86_64 2024-04-10T22:42:30.153 INFO:teuthology.orchestra.run.smithi040.stdout: kernel-modules-4.18.0-552.el8.x86_64 2024-04-10T22:42:30.153 INFO:teuthology.orchestra.run.smithi040.stdout: 2024-04-10T22:42:30.153 INFO:teuthology.orchestra.run.smithi040.stdout:Complete! 2024-04-10T22:42:30.244 INFO:teuthology.orchestra.run.smithi098.stdout: Running scriptlet: kernel-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:42:30.244 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : kernel-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:42:30.244 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : kernel-core-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:42:30.320 DEBUG:teuthology.orchestra.run.smithi040:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:42:30.545 INFO:teuthology.orchestra.run.smithi040.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-10T22:42:30.545 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-10T22:42:30.545 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-552.el8.x86_64 2024-04-10T22:42:30.545 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-10T22:42:30.546 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-10T22:42:30.546 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-04-10T22:42:30.546 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi040.front.sepia.ceph.com, path=None, version=distro) 2024-04-10T22:42:30.546 DEBUG:teuthology.orchestra.run.smithi040:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:42:30.724 INFO:teuthology.orchestra.run.smithi098.stdout: Verifying : kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:42:30.724 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-04-10T22:42:30.724 INFO:teuthology.orchestra.run.smithi098.stdout:Installed: 2024-04-10T22:42:30.724 INFO:teuthology.orchestra.run.smithi098.stdout: kernel-4.18.0-552.el8.x86_64 kernel-core-4.18.0-552.el8.x86_64 2024-04-10T22:42:30.724 INFO:teuthology.orchestra.run.smithi098.stdout: kernel-modules-4.18.0-552.el8.x86_64 2024-04-10T22:42:30.724 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-04-10T22:42:30.724 INFO:teuthology.orchestra.run.smithi098.stdout:Complete! 2024-04-10T22:42:30.767 INFO:teuthology.orchestra.run.smithi040.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-10T22:42:30.768 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-10T22:42:30.768 DEBUG:teuthology.orchestra.run.smithi040:> sudo rpm -qi grub2-tools 2024-04-10T22:42:30.803 INFO:teuthology.orchestra.run.smithi040.stdout:Name : grub2-tools 2024-04-10T22:42:30.803 INFO:teuthology.orchestra.run.smithi040.stdout:Epoch : 1 2024-04-10T22:42:30.803 INFO:teuthology.orchestra.run.smithi040.stdout:Version : 2.02 2024-04-10T22:42:30.803 INFO:teuthology.orchestra.run.smithi040.stdout:Release : 129.el8 2024-04-10T22:42:30.803 INFO:teuthology.orchestra.run.smithi040.stdout:Architecture: x86_64 2024-04-10T22:42:30.803 INFO:teuthology.orchestra.run.smithi040.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2024-04-10T22:42:30.803 INFO:teuthology.orchestra.run.smithi040.stdout:Group : System Environment/Base 2024-04-10T22:42:30.803 INFO:teuthology.orchestra.run.smithi040.stdout:Size : 9265586 2024-04-10T22:42:30.803 INFO:teuthology.orchestra.run.smithi040.stdout:License : GPLv3+ 2024-04-10T22:42:30.803 INFO:teuthology.orchestra.run.smithi040.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2024-04-10T22:42:30.803 INFO:teuthology.orchestra.run.smithi040.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2024-04-10T22:42:30.804 INFO:teuthology.orchestra.run.smithi040.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2024-04-10T22:42:30.804 INFO:teuthology.orchestra.run.smithi040.stdout:Build Host : kbuilder.bsys.centos.org 2024-04-10T22:42:30.804 INFO:teuthology.orchestra.run.smithi040.stdout:Relocations : (not relocatable) 2024-04-10T22:42:30.804 INFO:teuthology.orchestra.run.smithi040.stdout:Packager : CentOS BuildSystem 2024-04-10T22:42:30.804 INFO:teuthology.orchestra.run.smithi040.stdout:Vendor : CentOS 2024-04-10T22:42:30.804 INFO:teuthology.orchestra.run.smithi040.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-10T22:42:30.804 INFO:teuthology.orchestra.run.smithi040.stdout:Summary : Support tools for GRUB. 2024-04-10T22:42:30.804 INFO:teuthology.orchestra.run.smithi040.stdout:Description : 2024-04-10T22:42:30.804 INFO:teuthology.orchestra.run.smithi040.stdout: 2024-04-10T22:42:30.804 INFO:teuthology.orchestra.run.smithi040.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-10T22:42:30.804 INFO:teuthology.orchestra.run.smithi040.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-10T22:42:30.804 INFO:teuthology.orchestra.run.smithi040.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-10T22:42:30.804 INFO:teuthology.orchestra.run.smithi040.stdout:hardware devices. 2024-04-10T22:42:30.804 INFO:teuthology.orchestra.run.smithi040.stdout: 2024-04-10T22:42:30.804 INFO:teuthology.orchestra.run.smithi040.stdout:This subpackage provides tools for support of all platforms. 2024-04-10T22:42:30.805 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-10T22:42:30.805 INFO:teuthology.task.kernel:Updating grub on smithi040 to boot 4.18.0-552.el8.x86_64 2024-04-10T22:42:30.806 DEBUG:teuthology.orchestra.run.smithi040:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-10T22:42:30.856 DEBUG:teuthology.orchestra.run.smithi098:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:42:31.112 INFO:teuthology.orchestra.run.smithi098.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-10T22:42:31.112 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-10T22:42:31.112 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-552.el8.x86_64 2024-04-10T22:42:31.112 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-10T22:42:31.112 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-10T22:42:31.112 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-04-10T22:42:31.112 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi098.front.sepia.ceph.com, path=None, version=distro) 2024-04-10T22:42:31.113 DEBUG:teuthology.orchestra.run.smithi098:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:42:31.323 INFO:teuthology.orchestra.run.smithi098.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-10T22:42:31.323 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-10T22:42:31.323 DEBUG:teuthology.orchestra.run.smithi098:> sudo rpm -qi grub2-tools 2024-04-10T22:42:31.357 INFO:teuthology.orchestra.run.smithi098.stdout:Name : grub2-tools 2024-04-10T22:42:31.357 INFO:teuthology.orchestra.run.smithi098.stdout:Epoch : 1 2024-04-10T22:42:31.357 INFO:teuthology.orchestra.run.smithi098.stdout:Version : 2.02 2024-04-10T22:42:31.357 INFO:teuthology.orchestra.run.smithi098.stdout:Release : 129.el8 2024-04-10T22:42:31.357 INFO:teuthology.orchestra.run.smithi098.stdout:Architecture: x86_64 2024-04-10T22:42:31.357 INFO:teuthology.orchestra.run.smithi098.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2024-04-10T22:42:31.357 INFO:teuthology.orchestra.run.smithi098.stdout:Group : System Environment/Base 2024-04-10T22:42:31.357 INFO:teuthology.orchestra.run.smithi098.stdout:Size : 9265586 2024-04-10T22:42:31.357 INFO:teuthology.orchestra.run.smithi098.stdout:License : GPLv3+ 2024-04-10T22:42:31.357 INFO:teuthology.orchestra.run.smithi098.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2024-04-10T22:42:31.357 INFO:teuthology.orchestra.run.smithi098.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2024-04-10T22:42:31.358 INFO:teuthology.orchestra.run.smithi098.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2024-04-10T22:42:31.358 INFO:teuthology.orchestra.run.smithi098.stdout:Build Host : kbuilder.bsys.centos.org 2024-04-10T22:42:31.358 INFO:teuthology.orchestra.run.smithi098.stdout:Relocations : (not relocatable) 2024-04-10T22:42:31.358 INFO:teuthology.orchestra.run.smithi098.stdout:Packager : CentOS BuildSystem 2024-04-10T22:42:31.358 INFO:teuthology.orchestra.run.smithi098.stdout:Vendor : CentOS 2024-04-10T22:42:31.358 INFO:teuthology.orchestra.run.smithi098.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-10T22:42:31.358 INFO:teuthology.orchestra.run.smithi098.stdout:Summary : Support tools for GRUB. 2024-04-10T22:42:31.358 INFO:teuthology.orchestra.run.smithi098.stdout:Description : 2024-04-10T22:42:31.358 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-04-10T22:42:31.358 INFO:teuthology.orchestra.run.smithi098.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-10T22:42:31.358 INFO:teuthology.orchestra.run.smithi098.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-10T22:42:31.358 INFO:teuthology.orchestra.run.smithi098.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-10T22:42:31.358 INFO:teuthology.orchestra.run.smithi098.stdout:hardware devices. 2024-04-10T22:42:31.358 INFO:teuthology.orchestra.run.smithi098.stdout: 2024-04-10T22:42:31.359 INFO:teuthology.orchestra.run.smithi098.stdout:This subpackage provides tools for support of all platforms. 2024-04-10T22:42:31.359 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-10T22:42:31.360 INFO:teuthology.task.kernel:Updating grub on smithi098 to boot 4.18.0-552.el8.x86_64 2024-04-10T22:42:31.360 DEBUG:teuthology.orchestra.run.smithi098:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-10T22:42:32.439 INFO:teuthology.orchestra.run.smithi040.stderr:Generating grub configuration file ... 2024-04-10T22:42:33.088 INFO:teuthology.orchestra.run.smithi098.stderr:Generating grub configuration file ... 2024-04-10T22:42:34.133 INFO:teuthology.orchestra.run.smithi040.stderr:done 2024-04-10T22:42:34.135 DEBUG:teuthology.orchestra.run.smithi040:> mktemp 2024-04-10T22:42:34.160 INFO:teuthology.orchestra.run.smithi040.stdout:/tmp/tmp.wq2ySMJq3S 2024-04-10T22:42:34.160 DEBUG:teuthology.orchestra.run.smithi040:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.wq2ySMJq3S 2024-04-10T22:42:34.272 DEBUG:teuthology.orchestra.run.smithi040:> sudo chmod 0666 /tmp/tmp.wq2ySMJq3S 2024-04-10T22:42:34.378 DEBUG:teuthology.orchestra.remote:smithi040:/tmp/tmp.wq2ySMJq3S is 6KB 2024-04-10T22:42:34.426 DEBUG:teuthology.orchestra.run.smithi040:> rm -fr /tmp/tmp.wq2ySMJq3S 2024-04-10T22:42:34.444 DEBUG:teuthology.orchestra.run.smithi040:> sudo /bin/ls /boot/loader/entries || true 2024-04-10T22:42:34.519 INFO:teuthology.orchestra.run.smithi040.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2024-04-10T22:42:34.520 INFO:teuthology.orchestra.run.smithi040.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2024-04-10T22:42:34.520 INFO:teuthology.orchestra.run.smithi040.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2024-04-10T22:42:34.520 INFO:teuthology.orchestra.run.smithi040.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64.conf 2024-04-10T22:42:34.521 DEBUG:teuthology.orchestra.run.smithi040:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64 2024-04-10T22:42:34.676 DEBUG:teuthology.orchestra.run.smithi040:> sudo shutdown -r now 2024-04-10T22:42:34.976 INFO:teuthology.orchestra.run.smithi098.stderr:done 2024-04-10T22:42:34.977 DEBUG:teuthology.orchestra.run.smithi098:> mktemp 2024-04-10T22:42:35.007 INFO:teuthology.orchestra.run.smithi098.stdout:/tmp/tmp.YyKfgmveaa 2024-04-10T22:42:35.007 DEBUG:teuthology.orchestra.run.smithi098:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.YyKfgmveaa 2024-04-10T22:42:35.090 DEBUG:teuthology.orchestra.run.smithi098:> sudo chmod 0666 /tmp/tmp.YyKfgmveaa 2024-04-10T22:42:35.230 DEBUG:teuthology.orchestra.remote:smithi098:/tmp/tmp.YyKfgmveaa is 6KB 2024-04-10T22:42:35.279 DEBUG:teuthology.orchestra.run.smithi098:> rm -fr /tmp/tmp.YyKfgmveaa 2024-04-10T22:42:35.296 DEBUG:teuthology.orchestra.run.smithi098:> sudo /bin/ls /boot/loader/entries || true 2024-04-10T22:42:35.369 INFO:teuthology.orchestra.run.smithi098.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2024-04-10T22:42:35.370 INFO:teuthology.orchestra.run.smithi098.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2024-04-10T22:42:35.370 INFO:teuthology.orchestra.run.smithi098.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2024-04-10T22:42:35.370 INFO:teuthology.orchestra.run.smithi098.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64.conf 2024-04-10T22:42:35.371 DEBUG:teuthology.orchestra.run.smithi098:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64 2024-04-10T22:42:35.531 DEBUG:teuthology.orchestra.run.smithi098:> sudo shutdown -r now 2024-04-10T22:43:04.706 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-04-10T22:43:04.707 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi040.front.sepia.ceph.com' 2024-04-10T22:43:04.708 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi040.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:43:05.535 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-04-10T22:43:05.536 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi098.front.sepia.ceph.com' 2024-04-10T22:43:05.536 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi098.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:43:40.191 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.98 2024-04-10T22:43:49.198 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi098.front.sepia.ceph.com' 2024-04-10T22:43:49.200 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi098.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:44:04.754 DEBUG:teuthology.orchestra.remote:timed out 2024-04-10T22:44:07.582 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.98 2024-04-10T22:44:13.766 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi040.front.sepia.ceph.com' 2024-04-10T22:44:13.767 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi040.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:44:14.022 DEBUG:teuthology.orchestra.run.smithi040:> true 2024-04-10T22:44:14.356 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi040.front.sepia.ceph.com' 2024-04-10T22:44:14.356 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "4.18.0-552.el8.x86_64"... 2024-04-10T22:44:14.356 DEBUG:teuthology.orchestra.run.smithi040:> uname -r 2024-04-10T22:44:14.374 INFO:teuthology.orchestra.run.smithi040.stdout:4.18.0-552.el8.x86_64 2024-04-10T22:44:14.374 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-552.el8.x86_64 vs 4.18.0-552.el8.x86_64 2024-04-10T22:44:14.374 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-10T22:44:14.378 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-10T22:44:15.379 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-04-10T22:44:15.380 DEBUG:teuthology.orchestra.run.smithi040:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-10T22:44:15.450 INFO:teuthology.orchestra.run.smithi040.stdout:ttyS1 2024-04-10T22:44:15.467 DEBUG:teuthology.parallel:result is None 2024-04-10T22:44:19.584 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi098.front.sepia.ceph.com' 2024-04-10T22:44:19.585 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi098.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:44:19.906 DEBUG:teuthology.orchestra.run.smithi098:> true 2024-04-10T22:44:20.189 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi098.front.sepia.ceph.com' 2024-04-10T22:44:20.189 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "4.18.0-552.el8.x86_64"... 2024-04-10T22:44:20.189 DEBUG:teuthology.orchestra.run.smithi098:> uname -r 2024-04-10T22:44:20.206 INFO:teuthology.orchestra.run.smithi098.stdout:4.18.0-552.el8.x86_64 2024-04-10T22:44:20.206 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-552.el8.x86_64 vs 4.18.0-552.el8.x86_64 2024-04-10T22:44:20.206 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-10T22:44:20.206 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-10T22:44:21.208 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-04-10T22:44:21.208 DEBUG:teuthology.orchestra.run.smithi098:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-10T22:44:21.301 INFO:teuthology.orchestra.run.smithi098.stdout:ttyS1 2024-04-10T22:44:21.316 DEBUG:teuthology.parallel:result is None 2024-04-10T22:44:21.316 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-10T22:44:21.331 INFO:teuthology.task.internal:Creating test directory... 2024-04-10T22:44:21.332 DEBUG:teuthology.orchestra.run.smithi040:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-10T22:44:21.336 DEBUG:teuthology.orchestra.run.smithi098:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-10T22:44:21.357 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-10T22:44:21.364 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-10T22:44:21.374 INFO:teuthology.task.internal:Creating archive directory... 2024-04-10T22:44:21.374 DEBUG:teuthology.orchestra.run.smithi040:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-10T22:44:21.399 DEBUG:teuthology.orchestra.run.smithi098:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-10T22:44:21.448 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-10T22:44:21.455 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-10T22:44:21.455 DEBUG:teuthology.orchestra.run.smithi040:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-10T22:44:21.458 DEBUG:teuthology.orchestra.run.smithi098:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-10T22:44:21.500 INFO:teuthology.orchestra.run.smithi040.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:44:21.508 INFO:teuthology.orchestra.run.smithi098.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:44:21.512 INFO:teuthology.orchestra.run.smithi040.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:44:21.519 INFO:teuthology.orchestra.run.smithi098.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:44:21.520 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-10T22:44:21.527 INFO:teuthology.task.internal:Configuring sudo... 2024-04-10T22:44:21.527 DEBUG:teuthology.orchestra.run.smithi040:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-10T22:44:21.530 DEBUG:teuthology.orchestra.run.smithi098:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-10T22:44:21.559 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-10T22:44:21.571 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-10T22:44:21.571 DEBUG:teuthology.orchestra.run.smithi040:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-10T22:44:21.601 DEBUG:teuthology.orchestra.run.smithi098:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-10T22:44:21.619 DEBUG:teuthology.orchestra.run.smithi040:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:44:21.715 DEBUG:teuthology.orchestra.run.smithi040:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:44:21.766 DEBUG:teuthology.orchestra.run.smithi040:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:44:21.865 DEBUG:teuthology.orchestra.run.smithi040:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:44:21.898 DEBUG:teuthology.orchestra.run.smithi040:> set -ex 2024-04-10T22:44:21.899 DEBUG:teuthology.orchestra.run.smithi040:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-10T22:44:21.969 DEBUG:teuthology.orchestra.run.smithi098:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:44:22.023 DEBUG:teuthology.orchestra.run.smithi098:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:44:22.069 DEBUG:teuthology.orchestra.run.smithi098:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:44:22.172 DEBUG:teuthology.orchestra.run.smithi098:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:44:22.198 DEBUG:teuthology.orchestra.run.smithi098:> set -ex 2024-04-10T22:44:22.199 DEBUG:teuthology.orchestra.run.smithi098:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-10T22:44:22.265 DEBUG:teuthology.orchestra.run.smithi040:> sudo service rsyslog restart 2024-04-10T22:44:22.267 DEBUG:teuthology.orchestra.run.smithi098:> sudo service rsyslog restart 2024-04-10T22:44:22.336 INFO:teuthology.orchestra.run.smithi040.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:44:22.368 INFO:teuthology.orchestra.run.smithi098.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:44:22.773 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-10T22:44:22.781 INFO:teuthology.task.internal:Starting timer... 2024-04-10T22:44:22.781 INFO:teuthology.run_tasks:Running task pcp... 2024-04-10T22:44:22.852 INFO:teuthology.run_tasks:Running task selinux... 2024-04-10T22:44:22.861 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-04-10T22:44:22.862 DEBUG:teuthology.orchestra.run.smithi040:> sudo service auditd rotate 2024-04-10T22:44:22.955 INFO:teuthology.orchestra.run.smithi040.stdout:Rotating logs: 2024-04-10T22:44:22.957 DEBUG:teuthology.orchestra.run.smithi098:> sudo service auditd rotate 2024-04-10T22:44:23.065 INFO:teuthology.orchestra.run.smithi098.stdout:Rotating logs: 2024-04-10T22:44:23.067 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-10T22:44:23.067 DEBUG:teuthology.orchestra.run.smithi040:> /usr/sbin/getenforce 2024-04-10T22:44:23.090 INFO:teuthology.orchestra.run.smithi040.stdout:Permissive 2024-04-10T22:44:23.090 DEBUG:teuthology.orchestra.run.smithi098:> /usr/sbin/getenforce 2024-04-10T22:44:23.126 INFO:teuthology.orchestra.run.smithi098.stdout:Permissive 2024-04-10T22:44:23.126 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi040.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi098.front.sepia.ceph.com': 'permissive'} 2024-04-10T22:44:23.127 DEBUG:teuthology.orchestra.run.smithi040:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:44:23.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:44:23.159 DEBUG:teuthology.orchestra.run.smithi098:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:44:23.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:44:23.197 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-10T22:44:23.197 DEBUG:teuthology.orchestra.run.smithi040:> sudo /usr/sbin/setenforce permissive 2024-04-10T22:44:23.228 DEBUG:teuthology.orchestra.run.smithi098:> sudo /usr/sbin/setenforce permissive 2024-04-10T22:44:23.474 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-10T22:44:23.484 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-10T22:44:23.501 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-10T22:44:23.503 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi040.front.sepia.ceph.com,smithi098.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-10T22:55:10.175 INFO:teuthology.task.ansible:Archiving ansible failure log at: /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650179/ansible_failures.yaml 2024-04-10T22:55:10.177 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/__init__.py", line 123, in __enter__ self.begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 508, in begin super(CephLab, self).begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 342, in begin self.execute_playbook() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 374, in execute_playbook self._handle_failure(command, status) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 402, in _handle_failure raise AnsibleFailedError(failures) teuthology.exceptions.AnsibleFailedError: [',', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at'] 2024-04-10T22:55:10.420 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=759f047200c84c07aaa86a6c89010cda Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/__init__.py", line 123, in __enter__ self.begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 508, in begin super(CephLab, self).begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 342, in begin self.execute_playbook() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 374, in execute_playbook self._handle_failure(command, status) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 402, in _handle_failure raise AnsibleFailedError(failures) teuthology.exceptions.AnsibleFailedError: [',', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at'] 2024-04-10T22:55:10.422 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-10T22:55:10.431 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-10T22:55:10.431 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-10T22:55:10.489 DEBUG:teuthology.orchestra.run.smithi040:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-10T22:55:10.574 DEBUG:teuthology.orchestra.run.smithi098:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-10T22:55:10.639 DEBUG:teuthology.orchestra.run.smithi040:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:55:10.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:55:10.671 DEBUG:teuthology.orchestra.run.smithi098:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:55:10.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:55:10.702 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-10T22:55:10.714 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-10T22:55:10.722 INFO:teuthology.task.internal:Duration was 647.940212 seconds 2024-04-10T22:55:10.722 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-10T22:55:10.730 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-10T22:55:10.730 DEBUG:teuthology.orchestra.run.smithi040:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-10T22:55:10.732 DEBUG:teuthology.orchestra.run.smithi098:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-10T22:55:10.779 INFO:teuthology.orchestra.run.smithi040.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:55:10.790 INFO:teuthology.orchestra.run.smithi098.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:55:10.921 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-10T22:55:10.921 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi040.front.sepia.ceph.com 2024-04-10T22:55:10.921 DEBUG:teuthology.orchestra.run.smithi040:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-10T22:55:10.945 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi098.front.sepia.ceph.com 2024-04-10T22:55:10.945 DEBUG:teuthology.orchestra.run.smithi098:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-10T22:55:10.968 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-10T22:55:10.968 DEBUG:teuthology.orchestra.run.smithi040:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-10T22:55:10.988 DEBUG:teuthology.orchestra.run.smithi098:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-10T22:55:11.047 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-10T22:55:11.047 DEBUG:teuthology.orchestra.run.smithi040:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-10T22:55:11.069 DEBUG:teuthology.orchestra.run.smithi098:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-10T22:55:11.150 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-10T22:55:11.159 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-10T22:55:11.160 DEBUG:teuthology.orchestra.run.smithi040:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-10T22:55:11.187 DEBUG:teuthology.orchestra.run.smithi098:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-10T22:55:11.214 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-10T22:55:11.223 DEBUG:teuthology.orchestra.run.smithi040:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-10T22:55:11.231 DEBUG:teuthology.orchestra.run.smithi098:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-10T22:55:11.255 INFO:teuthology.orchestra.run.smithi040.stdout:kernel.core_pattern = core 2024-04-10T22:55:11.279 INFO:teuthology.orchestra.run.smithi098.stdout:kernel.core_pattern = core 2024-04-10T22:55:11.303 DEBUG:teuthology.orchestra.run.smithi040:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-10T22:55:11.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:55:11.334 DEBUG:teuthology.orchestra.run.smithi098:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-10T22:55:11.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:55:11.359 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-10T22:55:11.368 INFO:teuthology.task.internal:Transferring archived files... 2024-04-10T22:55:11.369 DEBUG:teuthology.misc:Transferring archived files from smithi040:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650179/remote/smithi040 2024-04-10T22:55:11.370 DEBUG:teuthology.orchestra.run.smithi040:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-10T22:55:11.436 DEBUG:teuthology.misc:Transferring archived files from smithi098:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650179/remote/smithi098 2024-04-10T22:55:11.437 DEBUG:teuthology.orchestra.run.smithi098:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-10T22:55:11.480 INFO:teuthology.task.internal:Removing archive directory... 2024-04-10T22:55:11.480 DEBUG:teuthology.orchestra.run.smithi040:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-10T22:55:11.483 DEBUG:teuthology.orchestra.run.smithi098:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-10T22:55:11.532 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-10T22:55:11.544 INFO:teuthology.task.internal:Not uploading archives. 2024-04-10T22:55:11.544 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-10T22:55:11.552 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-10T22:55:11.552 DEBUG:teuthology.orchestra.run.smithi040:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-10T22:55:11.554 DEBUG:teuthology.orchestra.run.smithi098:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-10T22:55:11.569 INFO:teuthology.orchestra.run.smithi040.stdout: 525355 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 10 22:55 /home/ubuntu/cephtest 2024-04-10T22:55:11.589 INFO:teuthology.orchestra.run.smithi098.stdout: 525830 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 10 22:55 /home/ubuntu/cephtest 2024-04-10T22:55:11.590 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-10T22:55:11.599 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-10T22:55:11.616 INFO:teuthology.nuke:Checking targets against current locks 2024-04-10T22:55:11.654 DEBUG:teuthology.nuke:shortname: smithi040 2024-04-10T22:55:11.655 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T22:55:11.702 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi040.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650179', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:15.067322', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLZ4p8PrKjn8Hebat7IWFTE+zKMwC2ccn2QbL10klKSaZqlrtnQn1GTdqW2/KJgD+Zp7kNXwEmkrNk+rQBTekPA='} 2024-04-10T22:55:11.717 DEBUG:teuthology.nuke:shortname: smithi098 2024-04-10T22:55:11.718 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T22:55:11.757 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi098.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650179', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:15.068846', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGZSZ0+yeaC3Dd+nqKVCgswtB7OhcXx+IPRijqlWeQAfHpmKnTWmoyifLK7J/WEniFf3eOCubEXwBHY4lYJe2us='} 2024-04-10T22:55:11.828 INFO:teuthology.orchestra.console.smithi040:Power off 2024-04-10T22:55:11.828 DEBUG:teuthology.orchestra.console.smithi040:pexpect command: ipmitool -H smithi040.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-10T22:55:11.864 INFO:teuthology.orchestra.console.smithi098:Power off 2024-04-10T22:55:11.864 DEBUG:teuthology.orchestra.console.smithi098:pexpect command: ipmitool -H smithi098.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-10T22:55:11.885 DEBUG:teuthology.orchestra.console.smithi040:power off output: Chassis Power Control: Down/Off 2024-04-10T22:55:11.885 DEBUG:teuthology.orchestra.console.smithi040:pexpect command: ipmitool -H smithi040.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:55:11.915 DEBUG:teuthology.orchestra.console.smithi098:power off output: Chassis Power Control: Down/Off 2024-04-10T22:55:11.916 DEBUG:teuthology.orchestra.console.smithi098:pexpect command: ipmitool -H smithi098.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:55:11.926 DEBUG:teuthology.orchestra.console.smithi040:check power output: Chassis Power is on 2024-04-10T22:55:11.969 DEBUG:teuthology.orchestra.console.smithi098:check power output: Chassis Power is on 2024-04-10T22:55:15.927 DEBUG:teuthology.orchestra.console.smithi040:pexpect command: ipmitool -H smithi040.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:55:15.970 DEBUG:teuthology.orchestra.console.smithi098:pexpect command: ipmitool -H smithi098.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:55:16.039 DEBUG:teuthology.orchestra.console.smithi040:check power output: Chassis Power is on 2024-04-10T22:55:16.081 DEBUG:teuthology.orchestra.console.smithi098:check power output: Chassis Power is on 2024-04-10T22:55:20.042 DEBUG:teuthology.orchestra.console.smithi040:pexpect command: ipmitool -H smithi040.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:55:20.082 DEBUG:teuthology.orchestra.console.smithi098:pexpect command: ipmitool -H smithi098.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:55:20.154 DEBUG:teuthology.orchestra.console.smithi040:check power output: Chassis Power is on 2024-04-10T22:55:20.193 DEBUG:teuthology.orchestra.console.smithi098:check power output: Chassis Power is on 2024-04-10T22:55:24.159 DEBUG:teuthology.orchestra.console.smithi040:pexpect command: ipmitool -H smithi040.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:55:24.194 DEBUG:teuthology.orchestra.console.smithi098:pexpect command: ipmitool -H smithi098.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:55:24.270 DEBUG:teuthology.orchestra.console.smithi040:check power output: Chassis Power is off 2024-04-10T22:55:24.304 DEBUG:teuthology.orchestra.console.smithi098:check power output: Chassis Power is off 2024-04-10T22:55:24.371 INFO:teuthology.orchestra.console.smithi040:Power off completed 2024-04-10T22:55:24.405 INFO:teuthology.orchestra.console.smithi098:Power off completed 2024-04-10T22:55:24.511 INFO:teuthology.run:Summary data: description: upgrade:pacific-x/stress-split/{0-distro/centos_8.stream_container_tools_crun 0-roles 1-start 2-first-half-tasks/snaps-few-objects 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} duration: 647.9402122497559 failure_reason: '['','', ''Bugzilla. In case of problems, contact the owner of this repository.'', ''Enabling a Copr repository. Please note that this repository is not part'', ''Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable'', ''Please do not file bug reports about these packages in Fedora'', ''The Fedora Project does not exercise any power over the contents of'', ''and packages are not held to any quality or security level.'', ''of the main distribution, and quality may vary.'', ''this repository beyond the rules outlined in the Copr FAQ at'']' owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=759f047200c84c07aaa86a6c89010cda status: dead success: false 2024-04-10T22:55:24.512 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-10T22:55:24.620 INFO:teuthology.run:DEAD