2024-04-10T22:37:39.562 INFO:root:teuthology version: 0.0.1.dev267+g6c63784 2024-04-10T22:37:39.563 DEBUG:teuthology.run:Teuthology command: teuthology --archive /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650178 --verbose --description upgrade:pacific-x/stress-split/{0-distro/centos_8.stream_container_tools 0-roles 1-start 2-first-half-tasks/readwrite 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} --name teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi --owner scheduled_teuthology@teuthology -- /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650178/orig.config.yaml 2024-04-10T22:37:39.618 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-10T22:37:39.706 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650178 branch: reef description: upgrade:pacific-x/stress-split/{0-distro/centos_8.stream_container_tools 0-roles 1-start 2-first-half-tasks/readwrite 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} email: ceph-qa@ceph.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' first_in_suite: false job_id: '7650178' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 8.stream overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: reef sha1: 4f70a9a90881107996c8b7a3e89ba8b32624713c owner: scheduled_teuthology@teuthology priority: 830 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 4604 sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: pacific clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 0/1 suite: upgrade:pacific-x suite_branch: reef suite_path: /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 4f70a9a90881107996c8b7a3e89ba8b32624713c targets: smithi136.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOIBWB8mA/E6Kk0FYy5EMCRIahUq+V86DI7qc4PsiCMNnJ4qgGmXK0jOXDVKBSAWd3IamgXCB++i20HjHJ6iifk= smithi155.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOm4scpyhYI54CYzQf4cN3kW49KVlup3fx8iiyzf5hYqx+x08KqUz3hzGeMbixTtO0QGv1ufyEBHJzS/9k8lqw8= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - install: branch: pacific exclude_packages: - ceph-volume - cephadm: cephadm_branch: pacific cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:pacific - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 6c637841c215537a4502385240412f1966e0faab timestamp: 2024-04-10_01:08:08 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-04-10T22:37:39.707 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa; will attempt to use it 2024-04-10T22:37:39.707 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa/tasks 2024-04-10T22:37:39.707 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-10T22:37:39.709 INFO:teuthology.task.internal:Checking packages... 2024-04-10T22:37:39.729 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'd540ebaca6b131a1dd560e7f69e024b133bbaa42' 2024-04-10T22:37:39.729 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-10T22:37:39.729 INFO:teuthology.packaging:ref: None 2024-04-10T22:37:39.729 INFO:teuthology.packaging:tag: None 2024-04-10T22:37:39.729 INFO:teuthology.packaging:branch: reef 2024-04-10T22:37:39.729 INFO:teuthology.packaging:sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 2024-04-10T22:37:39.729 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef 2024-04-10T22:37:40.055 INFO:teuthology.task.internal:Found packages for ceph version 18.2.2-1189.gc96c8d47 2024-04-10T22:37:40.056 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-10T22:37:40.063 INFO:teuthology.task.internal:no buildpackages task found 2024-04-10T22:37:40.063 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-10T22:37:40.082 INFO:teuthology.task.internal:Saving configuration 2024-04-10T22:37:40.103 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-10T22:37:40.109 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T22:37:40.133 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi136.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650178', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:14.656379', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOIBWB8mA/E6Kk0FYy5EMCRIahUq+V86DI7qc4PsiCMNnJ4qgGmXK0jOXDVKBSAWd3IamgXCB++i20HjHJ6iifk='} 2024-04-10T22:37:40.154 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi155.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650178', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:14.657682', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOm4scpyhYI54CYzQf4cN3kW49KVlup3fx8iiyzf5hYqx+x08KqUz3hzGeMbixTtO0QGv1ufyEBHJzS/9k8lqw8='} 2024-04-10T22:37:40.155 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-10T22:37:40.161 INFO:teuthology.task.internal:roles: ubuntu@smithi136.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-04-10T22:37:40.161 INFO:teuthology.task.internal:roles: ubuntu@smithi155.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-04-10T22:37:40.161 INFO:teuthology.run_tasks:Running task console_log... 2024-04-10T22:37:40.285 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fe82c2410d0>, signals=[15]) 2024-04-10T22:37:40.285 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-10T22:37:40.291 INFO:teuthology.task.internal:Opening connections... 2024-04-10T22:37:40.292 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi136.front.sepia.ceph.com 2024-04-10T22:37:40.294 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi136.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:37:40.370 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi155.front.sepia.ceph.com 2024-04-10T22:37:40.371 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi155.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:37:40.448 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-10T22:37:40.455 DEBUG:teuthology.orchestra.run.smithi136:> uname -m 2024-04-10T22:37:40.477 INFO:teuthology.orchestra.run.smithi136.stdout:x86_64 2024-04-10T22:37:40.477 DEBUG:teuthology.orchestra.run.smithi136:> cat /etc/os-release 2024-04-10T22:37:40.538 INFO:teuthology.orchestra.run.smithi136.stdout:NAME="CentOS Stream" 2024-04-10T22:37:40.539 INFO:teuthology.orchestra.run.smithi136.stdout:VERSION="8" 2024-04-10T22:37:40.539 INFO:teuthology.orchestra.run.smithi136.stdout:ID="centos" 2024-04-10T22:37:40.539 INFO:teuthology.orchestra.run.smithi136.stdout:ID_LIKE="rhel fedora" 2024-04-10T22:37:40.539 INFO:teuthology.orchestra.run.smithi136.stdout:VERSION_ID="8" 2024-04-10T22:37:40.539 INFO:teuthology.orchestra.run.smithi136.stdout:PLATFORM_ID="platform:el8" 2024-04-10T22:37:40.539 INFO:teuthology.orchestra.run.smithi136.stdout:PRETTY_NAME="CentOS Stream 8" 2024-04-10T22:37:40.539 INFO:teuthology.orchestra.run.smithi136.stdout:ANSI_COLOR="0;31" 2024-04-10T22:37:40.539 INFO:teuthology.orchestra.run.smithi136.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2024-04-10T22:37:40.539 INFO:teuthology.orchestra.run.smithi136.stdout:HOME_URL="https://centos.org/" 2024-04-10T22:37:40.539 INFO:teuthology.orchestra.run.smithi136.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-10T22:37:40.539 INFO:teuthology.orchestra.run.smithi136.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2024-04-10T22:37:40.539 INFO:teuthology.orchestra.run.smithi136.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-10T22:37:40.540 INFO:teuthology.lock.ops:Updating smithi136.front.sepia.ceph.com on lock server 2024-04-10T22:37:40.559 DEBUG:teuthology.orchestra.run.smithi155:> uname -m 2024-04-10T22:37:40.581 INFO:teuthology.orchestra.run.smithi155.stdout:x86_64 2024-04-10T22:37:40.582 DEBUG:teuthology.orchestra.run.smithi155:> cat /etc/os-release 2024-04-10T22:37:40.642 INFO:teuthology.orchestra.run.smithi155.stdout:NAME="CentOS Stream" 2024-04-10T22:37:40.642 INFO:teuthology.orchestra.run.smithi155.stdout:VERSION="8" 2024-04-10T22:37:40.642 INFO:teuthology.orchestra.run.smithi155.stdout:ID="centos" 2024-04-10T22:37:40.642 INFO:teuthology.orchestra.run.smithi155.stdout:ID_LIKE="rhel fedora" 2024-04-10T22:37:40.642 INFO:teuthology.orchestra.run.smithi155.stdout:VERSION_ID="8" 2024-04-10T22:37:40.642 INFO:teuthology.orchestra.run.smithi155.stdout:PLATFORM_ID="platform:el8" 2024-04-10T22:37:40.642 INFO:teuthology.orchestra.run.smithi155.stdout:PRETTY_NAME="CentOS Stream 8" 2024-04-10T22:37:40.642 INFO:teuthology.orchestra.run.smithi155.stdout:ANSI_COLOR="0;31" 2024-04-10T22:37:40.642 INFO:teuthology.orchestra.run.smithi155.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2024-04-10T22:37:40.642 INFO:teuthology.orchestra.run.smithi155.stdout:HOME_URL="https://centos.org/" 2024-04-10T22:37:40.642 INFO:teuthology.orchestra.run.smithi155.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-10T22:37:40.643 INFO:teuthology.orchestra.run.smithi155.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2024-04-10T22:37:40.643 INFO:teuthology.orchestra.run.smithi155.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-10T22:37:40.643 INFO:teuthology.lock.ops:Updating smithi155.front.sepia.ceph.com on lock server 2024-04-10T22:37:40.667 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-10T22:37:40.677 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-10T22:37:40.681 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-10T22:37:40.682 DEBUG:teuthology.orchestra.run.smithi136:> test '!' -e /home/ubuntu/cephtest 2024-04-10T22:37:40.684 DEBUG:teuthology.orchestra.run.smithi155:> test '!' -e /home/ubuntu/cephtest 2024-04-10T22:37:40.703 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-10T22:37:40.709 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-10T22:37:40.709 DEBUG:teuthology.orchestra.run.smithi136:> test -z $(ls -A /var/lib/ceph) 2024-04-10T22:37:40.746 DEBUG:teuthology.orchestra.run.smithi155:> test -z $(ls -A /var/lib/ceph) 2024-04-10T22:37:40.772 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-10T22:37:40.824 INFO:teuthology.run_tasks:Running task kernel... 2024-04-10T22:37:40.837 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-04-10T22:37:40.838 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-04-10T22:37:40.838 DEBUG:teuthology.orchestra.run.smithi136:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-10T22:37:40.838 DEBUG:teuthology.orchestra.run.smithi155:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-10T22:37:40.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:37:40.858 DEBUG:teuthology.orchestra.run.smithi155:> uname -r 2024-04-10T22:37:40.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:37:40.859 DEBUG:teuthology.orchestra.run.smithi136:> uname -r 2024-04-10T22:37:40.917 INFO:teuthology.orchestra.run.smithi155.stdout:4.18.0-489.el8.x86_64 2024-04-10T22:37:40.917 INFO:teuthology.task.kernel:Running kernel on smithi155: 4.18.0-489.el8.x86_64 2024-04-10T22:37:40.917 DEBUG:teuthology.orchestra.run.smithi155:> sudo yum install -y kernel 2024-04-10T22:37:40.920 INFO:teuthology.orchestra.run.smithi136.stdout:4.18.0-489.el8.x86_64 2024-04-10T22:37:40.920 INFO:teuthology.task.kernel:Running kernel on smithi136: 4.18.0-489.el8.x86_64 2024-04-10T22:37:40.920 DEBUG:teuthology.orchestra.run.smithi136:> sudo yum install -y kernel 2024-04-10T22:37:44.647 INFO:teuthology.orchestra.run.smithi136.stdout:CentOS Stream 8 - AppStream 24 MB/s | 28 MB 00:01 2024-04-10T22:39:56.048 INFO:teuthology.orchestra.run.smithi155.stdout:CentOS Stream 8 - AppStream 215 kB/s | 28 MB 02:12 2024-04-10T22:40:02.512 INFO:teuthology.orchestra.run.smithi136.stdout:CentOS Stream 8 - BaseOS 79 kB/s | 10 MB 02:12 2024-04-10T22:40:05.037 INFO:teuthology.orchestra.run.smithi136.stdout:CentOS Stream 8 - Extras 19 kB/s | 18 kB 00:00 2024-04-10T22:42:14.429 INFO:teuthology.orchestra.run.smithi155.stdout:CentOS Stream 8 - BaseOS 79 kB/s | 10 MB 02:13 2024-04-10T22:42:16.478 INFO:teuthology.orchestra.run.smithi136.stdout:CentOS Stream 8 - Extras common packages 60 B/s | 7.7 kB 02:11 2024-04-10T22:42:57.457 INFO:teuthology.orchestra.run.smithi136.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:40 2024-04-10T22:42:57.458 INFO:teuthology.orchestra.run.smithi136.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:42:57.458 INFO:teuthology.orchestra.run.smithi136.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:42:57.458 INFO:teuthology.orchestra.run.smithi136.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:42:58.733 INFO:teuthology.orchestra.run.smithi136.stdout:Extra Packages for Enterprise Linux 13 MB/s | 16 MB 00:01 2024-04-10T22:43:02.471 INFO:teuthology.orchestra.run.smithi136.stdout:lab-extras 423 kB/s | 24 kB 00:00 2024-04-10T22:43:02.612 INFO:teuthology.orchestra.run.smithi136.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:43:05.385 INFO:teuthology.orchestra.run.smithi136.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2024-04-10T22:43:05.385 INFO:teuthology.orchestra.run.smithi136.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2024-04-10T22:43:05.497 INFO:teuthology.orchestra.run.smithi136.stdout:Dependencies resolved. 2024-04-10T22:43:05.498 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2024-04-10T22:43:05.498 INFO:teuthology.orchestra.run.smithi136.stdout: Package Architecture Version Repository Size 2024-04-10T22:43:05.498 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2024-04-10T22:43:05.498 INFO:teuthology.orchestra.run.smithi136.stdout:Installing: 2024-04-10T22:43:05.498 INFO:teuthology.orchestra.run.smithi136.stdout: kernel x86_64 4.18.0-552.el8 baseos 10 M 2024-04-10T22:43:05.498 INFO:teuthology.orchestra.run.smithi136.stdout:Installing dependencies: 2024-04-10T22:43:05.498 INFO:teuthology.orchestra.run.smithi136.stdout: kernel-core x86_64 4.18.0-552.el8 baseos 44 M 2024-04-10T22:43:05.499 INFO:teuthology.orchestra.run.smithi136.stdout: kernel-modules x86_64 4.18.0-552.el8 baseos 36 M 2024-04-10T22:43:05.499 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-04-10T22:43:05.499 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction Summary 2024-04-10T22:43:05.499 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2024-04-10T22:43:05.499 INFO:teuthology.orchestra.run.smithi136.stdout:Install 3 Packages 2024-04-10T22:43:05.499 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-04-10T22:43:05.500 INFO:teuthology.orchestra.run.smithi136.stdout:Total download size: 90 M 2024-04-10T22:43:05.500 INFO:teuthology.orchestra.run.smithi136.stdout:Installed size: 96 M 2024-04-10T22:43:05.500 INFO:teuthology.orchestra.run.smithi136.stdout:Downloading Packages: 2024-04-10T22:43:08.089 INFO:teuthology.orchestra.run.smithi136.stdout:(1/3): kernel-4.18.0-552.el8.x86_64.rpm 4.4 MB/s | 10 MB 00:02 2024-04-10T22:43:15.221 INFO:teuthology.orchestra.run.smithi136.stdout:(2/3): kernel-modules-4.18.0-552.el8.x86_64.rpm 3.8 MB/s | 36 MB 00:09 2024-04-10T22:43:24.978 INFO:teuthology.orchestra.run.smithi136.stdout:(3/3): kernel-core-4.18.0-552.el8.x86_64.rpm 2.3 MB/s | 44 MB 00:19 2024-04-10T22:43:24.978 INFO:teuthology.orchestra.run.smithi136.stdout:-------------------------------------------------------------------------------- 2024-04-10T22:43:24.978 INFO:teuthology.orchestra.run.smithi136.stdout:Total 4.6 MB/s | 90 MB 00:19 2024-04-10T22:43:25.664 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction check 2024-04-10T22:43:26.095 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction check succeeded. 2024-04-10T22:43:26.095 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction test 2024-04-10T22:43:27.354 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction test succeeded. 2024-04-10T22:43:27.378 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction 2024-04-10T22:43:30.137 INFO:teuthology.orchestra.run.smithi136.stdout: Preparing : 1/1 2024-04-10T22:43:30.659 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:43:34.255 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:43:34.650 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:43:38.100 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:43:38.366 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : kernel-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:44:00.356 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:44:00.381 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:44:01.154 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: kernel-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:44:01.154 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : kernel-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:44:01.154 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : kernel-core-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:44:01.651 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:44:01.651 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-04-10T22:44:01.651 INFO:teuthology.orchestra.run.smithi136.stdout:Installed: 2024-04-10T22:44:01.651 INFO:teuthology.orchestra.run.smithi136.stdout: kernel-4.18.0-552.el8.x86_64 kernel-core-4.18.0-552.el8.x86_64 2024-04-10T22:44:01.651 INFO:teuthology.orchestra.run.smithi136.stdout: kernel-modules-4.18.0-552.el8.x86_64 2024-04-10T22:44:01.651 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-04-10T22:44:01.651 INFO:teuthology.orchestra.run.smithi136.stdout:Complete! 2024-04-10T22:44:01.790 DEBUG:teuthology.orchestra.run.smithi136:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:44:02.011 INFO:teuthology.orchestra.run.smithi136.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-10T22:44:02.011 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-10T22:44:02.011 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-552.el8.x86_64 2024-04-10T22:44:02.012 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-10T22:44:02.012 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-10T22:44:02.012 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-04-10T22:44:02.012 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi136.front.sepia.ceph.com, path=None, version=distro) 2024-04-10T22:44:02.012 DEBUG:teuthology.orchestra.run.smithi136:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:44:02.224 INFO:teuthology.orchestra.run.smithi136.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-10T22:44:02.224 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-10T22:44:02.224 DEBUG:teuthology.orchestra.run.smithi136:> sudo rpm -qi grub2-tools 2024-04-10T22:44:02.257 INFO:teuthology.orchestra.run.smithi136.stdout:Name : grub2-tools 2024-04-10T22:44:02.257 INFO:teuthology.orchestra.run.smithi136.stdout:Epoch : 1 2024-04-10T22:44:02.258 INFO:teuthology.orchestra.run.smithi136.stdout:Version : 2.02 2024-04-10T22:44:02.258 INFO:teuthology.orchestra.run.smithi136.stdout:Release : 129.el8 2024-04-10T22:44:02.258 INFO:teuthology.orchestra.run.smithi136.stdout:Architecture: x86_64 2024-04-10T22:44:02.258 INFO:teuthology.orchestra.run.smithi136.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2024-04-10T22:44:02.258 INFO:teuthology.orchestra.run.smithi136.stdout:Group : System Environment/Base 2024-04-10T22:44:02.258 INFO:teuthology.orchestra.run.smithi136.stdout:Size : 9265586 2024-04-10T22:44:02.258 INFO:teuthology.orchestra.run.smithi136.stdout:License : GPLv3+ 2024-04-10T22:44:02.258 INFO:teuthology.orchestra.run.smithi136.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2024-04-10T22:44:02.258 INFO:teuthology.orchestra.run.smithi136.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2024-04-10T22:44:02.258 INFO:teuthology.orchestra.run.smithi136.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2024-04-10T22:44:02.258 INFO:teuthology.orchestra.run.smithi136.stdout:Build Host : kbuilder.bsys.centos.org 2024-04-10T22:44:02.258 INFO:teuthology.orchestra.run.smithi136.stdout:Relocations : (not relocatable) 2024-04-10T22:44:02.259 INFO:teuthology.orchestra.run.smithi136.stdout:Packager : CentOS BuildSystem 2024-04-10T22:44:02.259 INFO:teuthology.orchestra.run.smithi136.stdout:Vendor : CentOS 2024-04-10T22:44:02.259 INFO:teuthology.orchestra.run.smithi136.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-10T22:44:02.259 INFO:teuthology.orchestra.run.smithi136.stdout:Summary : Support tools for GRUB. 2024-04-10T22:44:02.259 INFO:teuthology.orchestra.run.smithi136.stdout:Description : 2024-04-10T22:44:02.259 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-04-10T22:44:02.259 INFO:teuthology.orchestra.run.smithi136.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-10T22:44:02.259 INFO:teuthology.orchestra.run.smithi136.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-10T22:44:02.259 INFO:teuthology.orchestra.run.smithi136.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-10T22:44:02.259 INFO:teuthology.orchestra.run.smithi136.stdout:hardware devices. 2024-04-10T22:44:02.259 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-04-10T22:44:02.259 INFO:teuthology.orchestra.run.smithi136.stdout:This subpackage provides tools for support of all platforms. 2024-04-10T22:44:02.260 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-10T22:44:02.260 INFO:teuthology.task.kernel:Updating grub on smithi136 to boot 4.18.0-552.el8.x86_64 2024-04-10T22:44:02.260 DEBUG:teuthology.orchestra.run.smithi136:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-10T22:44:03.661 INFO:teuthology.orchestra.run.smithi136.stderr:Generating grub configuration file ... 2024-04-10T22:44:05.616 INFO:teuthology.orchestra.run.smithi136.stderr:done 2024-04-10T22:44:05.618 DEBUG:teuthology.orchestra.run.smithi136:> mktemp 2024-04-10T22:44:05.640 INFO:teuthology.orchestra.run.smithi136.stdout:/tmp/tmp.NrfnhHQLfp 2024-04-10T22:44:05.641 DEBUG:teuthology.orchestra.run.smithi136:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.NrfnhHQLfp 2024-04-10T22:44:05.714 DEBUG:teuthology.orchestra.run.smithi136:> sudo chmod 0666 /tmp/tmp.NrfnhHQLfp 2024-04-10T22:44:05.857 DEBUG:teuthology.orchestra.remote:smithi136:/tmp/tmp.NrfnhHQLfp is 6KB 2024-04-10T22:44:05.906 DEBUG:teuthology.orchestra.run.smithi136:> rm -fr /tmp/tmp.NrfnhHQLfp 2024-04-10T22:44:05.924 DEBUG:teuthology.orchestra.run.smithi136:> sudo /bin/ls /boot/loader/entries || true 2024-04-10T22:44:06.002 INFO:teuthology.orchestra.run.smithi136.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2024-04-10T22:44:06.002 INFO:teuthology.orchestra.run.smithi136.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2024-04-10T22:44:06.002 INFO:teuthology.orchestra.run.smithi136.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2024-04-10T22:44:06.002 INFO:teuthology.orchestra.run.smithi136.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64.conf 2024-04-10T22:44:06.003 DEBUG:teuthology.orchestra.run.smithi136:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64 2024-04-10T22:44:06.171 DEBUG:teuthology.orchestra.run.smithi136:> sudo shutdown -r now 2024-04-10T22:44:25.636 INFO:teuthology.orchestra.run.smithi155.stdout:CentOS Stream 8 - Extras 141 B/s | 18 kB 02:09 2024-04-10T22:44:26.095 INFO:teuthology.orchestra.run.smithi155.stdout:CentOS Stream 8 - Extras common packages 22 kB/s | 7.7 kB 00:00 2024-04-10T22:44:36.173 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-04-10T22:44:36.174 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi136.front.sepia.ceph.com' 2024-04-10T22:44:36.175 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi136.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:45:08.477 INFO:teuthology.orchestra.run.smithi155.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:42 2024-04-10T22:45:08.478 INFO:teuthology.orchestra.run.smithi155.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:45:08.478 INFO:teuthology.orchestra.run.smithi155.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:45:08.478 INFO:teuthology.orchestra.run.smithi155.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:45:10.193 INFO:teuthology.orchestra.run.smithi155.stdout:Extra Packages for Enterprise Linux 9.5 MB/s | 16 MB 00:01 2024-04-10T22:45:10.814 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.136 2024-04-10T22:45:13.983 INFO:teuthology.orchestra.run.smithi155.stdout:lab-extras 403 kB/s | 24 kB 00:00 2024-04-10T22:45:14.124 INFO:teuthology.orchestra.run.smithi155.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:45:16.954 INFO:teuthology.orchestra.run.smithi155.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2024-04-10T22:45:16.955 INFO:teuthology.orchestra.run.smithi155.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2024-04-10T22:45:17.081 INFO:teuthology.orchestra.run.smithi155.stdout:Dependencies resolved. 2024-04-10T22:45:17.082 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2024-04-10T22:45:17.082 INFO:teuthology.orchestra.run.smithi155.stdout: Package Architecture Version Repository Size 2024-04-10T22:45:17.082 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2024-04-10T22:45:17.082 INFO:teuthology.orchestra.run.smithi155.stdout:Installing: 2024-04-10T22:45:17.082 INFO:teuthology.orchestra.run.smithi155.stdout: kernel x86_64 4.18.0-552.el8 baseos 10 M 2024-04-10T22:45:17.083 INFO:teuthology.orchestra.run.smithi155.stdout:Installing dependencies: 2024-04-10T22:45:17.083 INFO:teuthology.orchestra.run.smithi155.stdout: kernel-core x86_64 4.18.0-552.el8 baseos 44 M 2024-04-10T22:45:17.083 INFO:teuthology.orchestra.run.smithi155.stdout: kernel-modules x86_64 4.18.0-552.el8 baseos 36 M 2024-04-10T22:45:17.083 INFO:teuthology.orchestra.run.smithi155.stdout: 2024-04-10T22:45:17.083 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction Summary 2024-04-10T22:45:17.083 INFO:teuthology.orchestra.run.smithi155.stdout:================================================================================ 2024-04-10T22:45:17.083 INFO:teuthology.orchestra.run.smithi155.stdout:Install 3 Packages 2024-04-10T22:45:17.083 INFO:teuthology.orchestra.run.smithi155.stdout: 2024-04-10T22:45:17.084 INFO:teuthology.orchestra.run.smithi155.stdout:Total download size: 90 M 2024-04-10T22:45:17.084 INFO:teuthology.orchestra.run.smithi155.stdout:Installed size: 96 M 2024-04-10T22:45:17.084 INFO:teuthology.orchestra.run.smithi155.stdout:Downloading Packages: 2024-04-10T22:45:19.815 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi136.front.sepia.ceph.com' 2024-04-10T22:45:19.816 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi136.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:45:20.759 INFO:teuthology.orchestra.run.smithi155.stdout:(1/3): kernel-4.18.0-552.el8.x86_64.rpm 3.0 MB/s | 10 MB 00:03 2024-04-10T22:45:32.074 INFO:teuthology.orchestra.run.smithi155.stdout:(2/3): kernel-modules-4.18.0-552.el8.x86_64.rpm 2.5 MB/s | 36 MB 00:14 2024-04-10T22:45:36.865 INFO:teuthology.orchestra.run.smithi155.stdout:(3/3): kernel-core-4.18.0-552.el8.x86_64.rpm 2.2 MB/s | 44 MB 00:19 2024-04-10T22:45:36.866 INFO:teuthology.orchestra.run.smithi155.stdout:-------------------------------------------------------------------------------- 2024-04-10T22:45:36.866 INFO:teuthology.orchestra.run.smithi155.stdout:Total 4.6 MB/s | 90 MB 00:19 2024-04-10T22:45:37.551 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction check 2024-04-10T22:45:37.986 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction check succeeded. 2024-04-10T22:45:37.987 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction test 2024-04-10T22:45:38.206 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.136 2024-04-10T22:45:39.244 INFO:teuthology.orchestra.run.smithi155.stdout:Transaction test succeeded. 2024-04-10T22:45:39.269 INFO:teuthology.orchestra.run.smithi155.stdout:Running transaction 2024-04-10T22:45:42.072 INFO:teuthology.orchestra.run.smithi155.stdout: Preparing : 1/1 2024-04-10T22:45:42.673 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:45:46.288 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:45:46.673 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:45:50.111 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:45:50.207 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi136.front.sepia.ceph.com' 2024-04-10T22:45:50.208 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi136.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:45:50.420 INFO:teuthology.orchestra.run.smithi155.stdout: Installing : kernel-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:45:50.697 DEBUG:teuthology.orchestra.run.smithi136:> true 2024-04-10T22:45:51.015 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi136.front.sepia.ceph.com' 2024-04-10T22:45:51.015 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "4.18.0-552.el8.x86_64"... 2024-04-10T22:45:51.016 DEBUG:teuthology.orchestra.run.smithi136:> uname -r 2024-04-10T22:45:51.033 INFO:teuthology.orchestra.run.smithi136.stdout:4.18.0-552.el8.x86_64 2024-04-10T22:45:51.033 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-552.el8.x86_64 vs 4.18.0-552.el8.x86_64 2024-04-10T22:45:51.033 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-10T22:45:51.033 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-10T22:45:52.034 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-04-10T22:45:52.035 DEBUG:teuthology.orchestra.run.smithi136:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-10T22:45:52.134 INFO:teuthology.orchestra.run.smithi136.stdout:ttyS1 2024-04-10T22:45:52.150 DEBUG:teuthology.parallel:result is None 2024-04-10T22:46:12.415 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:46:12.437 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:46:12.779 INFO:teuthology.orchestra.run.smithi155.stdout: Running scriptlet: kernel-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:46:12.779 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : kernel-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:46:12.779 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : kernel-core-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:46:13.272 INFO:teuthology.orchestra.run.smithi155.stdout: Verifying : kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:46:13.272 INFO:teuthology.orchestra.run.smithi155.stdout: 2024-04-10T22:46:13.272 INFO:teuthology.orchestra.run.smithi155.stdout:Installed: 2024-04-10T22:46:13.272 INFO:teuthology.orchestra.run.smithi155.stdout: kernel-4.18.0-552.el8.x86_64 kernel-core-4.18.0-552.el8.x86_64 2024-04-10T22:46:13.273 INFO:teuthology.orchestra.run.smithi155.stdout: kernel-modules-4.18.0-552.el8.x86_64 2024-04-10T22:46:13.273 INFO:teuthology.orchestra.run.smithi155.stdout: 2024-04-10T22:46:13.273 INFO:teuthology.orchestra.run.smithi155.stdout:Complete! 2024-04-10T22:46:13.405 DEBUG:teuthology.orchestra.run.smithi155:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:46:13.625 INFO:teuthology.orchestra.run.smithi155.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-10T22:46:13.626 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-10T22:46:13.626 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-552.el8.x86_64 2024-04-10T22:46:13.626 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-10T22:46:13.626 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-10T22:46:13.626 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-04-10T22:46:13.626 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi155.front.sepia.ceph.com, path=None, version=distro) 2024-04-10T22:46:13.627 DEBUG:teuthology.orchestra.run.smithi155:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:46:13.839 INFO:teuthology.orchestra.run.smithi155.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-10T22:46:13.839 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-10T22:46:13.839 DEBUG:teuthology.orchestra.run.smithi155:> sudo rpm -qi grub2-tools 2024-04-10T22:46:13.874 INFO:teuthology.orchestra.run.smithi155.stdout:Name : grub2-tools 2024-04-10T22:46:13.874 INFO:teuthology.orchestra.run.smithi155.stdout:Epoch : 1 2024-04-10T22:46:13.874 INFO:teuthology.orchestra.run.smithi155.stdout:Version : 2.02 2024-04-10T22:46:13.874 INFO:teuthology.orchestra.run.smithi155.stdout:Release : 129.el8 2024-04-10T22:46:13.874 INFO:teuthology.orchestra.run.smithi155.stdout:Architecture: x86_64 2024-04-10T22:46:13.874 INFO:teuthology.orchestra.run.smithi155.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2024-04-10T22:46:13.874 INFO:teuthology.orchestra.run.smithi155.stdout:Group : System Environment/Base 2024-04-10T22:46:13.874 INFO:teuthology.orchestra.run.smithi155.stdout:Size : 9265586 2024-04-10T22:46:13.874 INFO:teuthology.orchestra.run.smithi155.stdout:License : GPLv3+ 2024-04-10T22:46:13.874 INFO:teuthology.orchestra.run.smithi155.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2024-04-10T22:46:13.874 INFO:teuthology.orchestra.run.smithi155.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2024-04-10T22:46:13.875 INFO:teuthology.orchestra.run.smithi155.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2024-04-10T22:46:13.875 INFO:teuthology.orchestra.run.smithi155.stdout:Build Host : kbuilder.bsys.centos.org 2024-04-10T22:46:13.875 INFO:teuthology.orchestra.run.smithi155.stdout:Relocations : (not relocatable) 2024-04-10T22:46:13.875 INFO:teuthology.orchestra.run.smithi155.stdout:Packager : CentOS BuildSystem 2024-04-10T22:46:13.875 INFO:teuthology.orchestra.run.smithi155.stdout:Vendor : CentOS 2024-04-10T22:46:13.875 INFO:teuthology.orchestra.run.smithi155.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-10T22:46:13.875 INFO:teuthology.orchestra.run.smithi155.stdout:Summary : Support tools for GRUB. 2024-04-10T22:46:13.875 INFO:teuthology.orchestra.run.smithi155.stdout:Description : 2024-04-10T22:46:13.875 INFO:teuthology.orchestra.run.smithi155.stdout: 2024-04-10T22:46:13.875 INFO:teuthology.orchestra.run.smithi155.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-10T22:46:13.875 INFO:teuthology.orchestra.run.smithi155.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-10T22:46:13.875 INFO:teuthology.orchestra.run.smithi155.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-10T22:46:13.875 INFO:teuthology.orchestra.run.smithi155.stdout:hardware devices. 2024-04-10T22:46:13.875 INFO:teuthology.orchestra.run.smithi155.stdout: 2024-04-10T22:46:13.876 INFO:teuthology.orchestra.run.smithi155.stdout:This subpackage provides tools for support of all platforms. 2024-04-10T22:46:13.876 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-10T22:46:13.877 INFO:teuthology.task.kernel:Updating grub on smithi155 to boot 4.18.0-552.el8.x86_64 2024-04-10T22:46:13.877 DEBUG:teuthology.orchestra.run.smithi155:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-10T22:46:15.534 INFO:teuthology.orchestra.run.smithi155.stderr:Generating grub configuration file ... 2024-04-10T22:46:17.939 INFO:teuthology.orchestra.run.smithi155.stderr:done 2024-04-10T22:46:17.941 DEBUG:teuthology.orchestra.run.smithi155:> mktemp 2024-04-10T22:46:17.964 INFO:teuthology.orchestra.run.smithi155.stdout:/tmp/tmp.ggP0Cgyliy 2024-04-10T22:46:17.964 DEBUG:teuthology.orchestra.run.smithi155:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.ggP0Cgyliy 2024-04-10T22:46:18.038 DEBUG:teuthology.orchestra.run.smithi155:> sudo chmod 0666 /tmp/tmp.ggP0Cgyliy 2024-04-10T22:46:18.191 DEBUG:teuthology.orchestra.remote:smithi155:/tmp/tmp.ggP0Cgyliy is 6KB 2024-04-10T22:46:18.239 DEBUG:teuthology.orchestra.run.smithi155:> rm -fr /tmp/tmp.ggP0Cgyliy 2024-04-10T22:46:18.257 DEBUG:teuthology.orchestra.run.smithi155:> sudo /bin/ls /boot/loader/entries || true 2024-04-10T22:46:18.335 INFO:teuthology.orchestra.run.smithi155.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2024-04-10T22:46:18.336 INFO:teuthology.orchestra.run.smithi155.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2024-04-10T22:46:18.336 INFO:teuthology.orchestra.run.smithi155.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2024-04-10T22:46:18.336 INFO:teuthology.orchestra.run.smithi155.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64.conf 2024-04-10T22:46:18.337 DEBUG:teuthology.orchestra.run.smithi155:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64 2024-04-10T22:46:18.487 DEBUG:teuthology.orchestra.run.smithi155:> sudo shutdown -r now 2024-04-10T22:46:48.491 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-04-10T22:46:48.492 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi155.front.sepia.ceph.com' 2024-04-10T22:46:48.493 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi155.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:47:23.171 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.155 2024-04-10T22:47:32.172 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi155.front.sepia.ceph.com' 2024-04-10T22:47:32.173 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi155.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:47:50.562 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.155 2024-04-10T22:48:02.564 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi155.front.sepia.ceph.com' 2024-04-10T22:48:02.565 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi155.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:48:03.213 DEBUG:teuthology.orchestra.run.smithi155:> true 2024-04-10T22:48:03.550 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi155.front.sepia.ceph.com' 2024-04-10T22:48:03.553 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "4.18.0-552.el8.x86_64"... 2024-04-10T22:48:03.553 DEBUG:teuthology.orchestra.run.smithi155:> uname -r 2024-04-10T22:48:03.570 INFO:teuthology.orchestra.run.smithi155.stdout:4.18.0-552.el8.x86_64 2024-04-10T22:48:03.570 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-552.el8.x86_64 vs 4.18.0-552.el8.x86_64 2024-04-10T22:48:03.570 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-10T22:48:03.571 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-10T22:48:04.572 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-04-10T22:48:04.572 DEBUG:teuthology.orchestra.run.smithi155:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-10T22:48:04.646 INFO:teuthology.orchestra.run.smithi155.stdout:ttyS1 2024-04-10T22:48:04.662 DEBUG:teuthology.parallel:result is None 2024-04-10T22:48:04.662 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-10T22:48:04.669 INFO:teuthology.task.internal:Creating test directory... 2024-04-10T22:48:04.669 DEBUG:teuthology.orchestra.run.smithi136:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-10T22:48:04.673 DEBUG:teuthology.orchestra.run.smithi155:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-10T22:48:04.694 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-10T22:48:04.702 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-10T22:48:04.713 INFO:teuthology.task.internal:Creating archive directory... 2024-04-10T22:48:04.714 DEBUG:teuthology.orchestra.run.smithi136:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-10T22:48:04.738 DEBUG:teuthology.orchestra.run.smithi155:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-10T22:48:04.775 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-10T22:48:04.786 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-10T22:48:04.786 DEBUG:teuthology.orchestra.run.smithi136:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-10T22:48:04.811 DEBUG:teuthology.orchestra.run.smithi155:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-10T22:48:04.853 INFO:teuthology.orchestra.run.smithi136.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:48:04.864 INFO:teuthology.orchestra.run.smithi136.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:48:04.864 INFO:teuthology.orchestra.run.smithi155.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:48:04.874 INFO:teuthology.orchestra.run.smithi155.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:48:04.876 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-10T22:48:04.884 INFO:teuthology.task.internal:Configuring sudo... 2024-04-10T22:48:04.884 DEBUG:teuthology.orchestra.run.smithi136:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-10T22:48:04.886 DEBUG:teuthology.orchestra.run.smithi155:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-10T22:48:04.914 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-10T22:48:04.926 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-10T22:48:04.926 DEBUG:teuthology.orchestra.run.smithi136:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-10T22:48:04.957 DEBUG:teuthology.orchestra.run.smithi155:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-10T22:48:04.974 DEBUG:teuthology.orchestra.run.smithi136:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:48:05.061 DEBUG:teuthology.orchestra.run.smithi136:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:48:05.110 DEBUG:teuthology.orchestra.run.smithi136:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:48:05.191 DEBUG:teuthology.orchestra.run.smithi136:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:48:05.237 DEBUG:teuthology.orchestra.run.smithi136:> set -ex 2024-04-10T22:48:05.237 DEBUG:teuthology.orchestra.run.smithi136:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-10T22:48:05.303 DEBUG:teuthology.orchestra.run.smithi155:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:48:05.347 DEBUG:teuthology.orchestra.run.smithi155:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:48:05.392 DEBUG:teuthology.orchestra.run.smithi155:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:48:05.476 DEBUG:teuthology.orchestra.run.smithi155:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:48:05.513 DEBUG:teuthology.orchestra.run.smithi155:> set -ex 2024-04-10T22:48:05.513 DEBUG:teuthology.orchestra.run.smithi155:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-10T22:48:05.578 DEBUG:teuthology.orchestra.run.smithi136:> sudo service rsyslog restart 2024-04-10T22:48:05.581 DEBUG:teuthology.orchestra.run.smithi155:> sudo service rsyslog restart 2024-04-10T22:48:05.635 INFO:teuthology.orchestra.run.smithi136.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:48:05.679 INFO:teuthology.orchestra.run.smithi155.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:48:05.881 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-10T22:48:05.889 INFO:teuthology.task.internal:Starting timer... 2024-04-10T22:48:05.889 INFO:teuthology.run_tasks:Running task pcp... 2024-04-10T22:48:05.905 INFO:teuthology.run_tasks:Running task selinux... 2024-04-10T22:48:05.916 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-04-10T22:48:05.916 DEBUG:teuthology.orchestra.run.smithi136:> sudo service auditd rotate 2024-04-10T22:48:06.011 INFO:teuthology.orchestra.run.smithi136.stdout:Rotating logs: 2024-04-10T22:48:06.012 DEBUG:teuthology.orchestra.run.smithi155:> sudo service auditd rotate 2024-04-10T22:48:06.092 INFO:teuthology.orchestra.run.smithi155.stdout:Rotating logs: 2024-04-10T22:48:06.094 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-10T22:48:06.094 DEBUG:teuthology.orchestra.run.smithi136:> /usr/sbin/getenforce 2024-04-10T22:48:06.143 INFO:teuthology.orchestra.run.smithi136.stdout:Permissive 2024-04-10T22:48:06.143 DEBUG:teuthology.orchestra.run.smithi155:> /usr/sbin/getenforce 2024-04-10T22:48:06.164 INFO:teuthology.orchestra.run.smithi155.stdout:Permissive 2024-04-10T22:48:06.165 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi136.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi155.front.sepia.ceph.com': 'permissive'} 2024-04-10T22:48:06.165 DEBUG:teuthology.orchestra.run.smithi136:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:48:06.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:48:06.192 DEBUG:teuthology.orchestra.run.smithi155:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:48:06.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:48:06.228 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-10T22:48:06.228 DEBUG:teuthology.orchestra.run.smithi136:> sudo /usr/sbin/setenforce permissive 2024-04-10T22:48:06.259 DEBUG:teuthology.orchestra.run.smithi155:> sudo /usr/sbin/setenforce permissive 2024-04-10T22:48:06.324 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-10T22:48:06.334 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-10T22:48:06.351 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-10T22:48:06.352 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi136.front.sepia.ceph.com,smithi155.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-10T23:03:09.918 INFO:teuthology.task.ansible:Archiving ansible failure log at: /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650178/ansible_failures.yaml 2024-04-10T23:03:09.920 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/__init__.py", line 123, in __enter__ self.begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 508, in begin super(CephLab, self).begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 342, in begin self.execute_playbook() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 374, in execute_playbook self._handle_failure(command, status) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 402, in _handle_failure raise AnsibleFailedError(failures) teuthology.exceptions.AnsibleFailedError: [',', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at'] 2024-04-10T23:03:10.160 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=5dbbd5d1ed484f8ca6195d8f0b0e8ce3 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/__init__.py", line 123, in __enter__ self.begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 508, in begin super(CephLab, self).begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 342, in begin self.execute_playbook() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 374, in execute_playbook self._handle_failure(command, status) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 402, in _handle_failure raise AnsibleFailedError(failures) teuthology.exceptions.AnsibleFailedError: [',', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at'] 2024-04-10T23:03:10.162 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-10T23:03:10.170 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-10T23:03:10.171 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-10T23:03:10.194 DEBUG:teuthology.orchestra.run.smithi136:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-10T23:03:10.282 DEBUG:teuthology.orchestra.run.smithi155:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-10T23:03:10.349 DEBUG:teuthology.orchestra.run.smithi136:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T23:03:10.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T23:03:10.381 DEBUG:teuthology.orchestra.run.smithi155:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T23:03:10.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T23:03:10.412 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-10T23:03:10.421 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-10T23:03:10.428 INFO:teuthology.task.internal:Duration was 904.538946 seconds 2024-04-10T23:03:10.428 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-10T23:03:10.437 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-10T23:03:10.437 DEBUG:teuthology.orchestra.run.smithi136:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-10T23:03:10.439 DEBUG:teuthology.orchestra.run.smithi155:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-10T23:03:10.485 INFO:teuthology.orchestra.run.smithi136.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T23:03:10.500 INFO:teuthology.orchestra.run.smithi155.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T23:03:10.723 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-10T23:03:10.724 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi136.front.sepia.ceph.com 2024-04-10T23:03:10.724 DEBUG:teuthology.orchestra.run.smithi136:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-10T23:03:10.747 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi155.front.sepia.ceph.com 2024-04-10T23:03:10.748 DEBUG:teuthology.orchestra.run.smithi155:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-10T23:03:10.770 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-10T23:03:10.770 DEBUG:teuthology.orchestra.run.smithi136:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-10T23:03:10.790 DEBUG:teuthology.orchestra.run.smithi155:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-10T23:03:10.850 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-10T23:03:10.851 DEBUG:teuthology.orchestra.run.smithi136:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-10T23:03:10.870 DEBUG:teuthology.orchestra.run.smithi155:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-10T23:03:10.953 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-10T23:03:10.962 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-10T23:03:10.962 DEBUG:teuthology.orchestra.run.smithi136:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-10T23:03:10.991 DEBUG:teuthology.orchestra.run.smithi155:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-10T23:03:11.018 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-10T23:03:11.028 DEBUG:teuthology.orchestra.run.smithi136:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-10T23:03:11.034 DEBUG:teuthology.orchestra.run.smithi155:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-10T23:03:11.057 INFO:teuthology.orchestra.run.smithi136.stdout:kernel.core_pattern = core 2024-04-10T23:03:11.084 INFO:teuthology.orchestra.run.smithi155.stdout:kernel.core_pattern = core 2024-04-10T23:03:11.103 DEBUG:teuthology.orchestra.run.smithi136:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-10T23:03:11.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T23:03:11.135 DEBUG:teuthology.orchestra.run.smithi155:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-10T23:03:11.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T23:03:11.161 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-10T23:03:11.170 INFO:teuthology.task.internal:Transferring archived files... 2024-04-10T23:03:11.172 DEBUG:teuthology.misc:Transferring archived files from smithi136:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650178/remote/smithi136 2024-04-10T23:03:11.173 DEBUG:teuthology.orchestra.run.smithi136:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-10T23:03:11.225 DEBUG:teuthology.misc:Transferring archived files from smithi155:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650178/remote/smithi155 2024-04-10T23:03:11.225 DEBUG:teuthology.orchestra.run.smithi155:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-10T23:03:11.271 INFO:teuthology.task.internal:Removing archive directory... 2024-04-10T23:03:11.271 DEBUG:teuthology.orchestra.run.smithi136:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-10T23:03:11.274 DEBUG:teuthology.orchestra.run.smithi155:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-10T23:03:11.324 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-10T23:03:11.339 INFO:teuthology.task.internal:Not uploading archives. 2024-04-10T23:03:11.340 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-10T23:03:11.347 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-10T23:03:11.348 DEBUG:teuthology.orchestra.run.smithi136:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-10T23:03:11.350 DEBUG:teuthology.orchestra.run.smithi155:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-10T23:03:11.365 INFO:teuthology.orchestra.run.smithi136.stdout: 393224 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 10 23:03 /home/ubuntu/cephtest 2024-04-10T23:03:11.381 INFO:teuthology.orchestra.run.smithi155.stdout: 525649 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 10 23:03 /home/ubuntu/cephtest 2024-04-10T23:03:11.382 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-10T23:03:11.410 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-10T23:03:11.480 INFO:teuthology.nuke:Checking targets against current locks 2024-04-10T23:03:11.526 DEBUG:teuthology.nuke:shortname: smithi136 2024-04-10T23:03:11.527 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T23:03:11.573 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi136.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650178', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:14.656379', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOIBWB8mA/E6Kk0FYy5EMCRIahUq+V86DI7qc4PsiCMNnJ4qgGmXK0jOXDVKBSAWd3IamgXCB++i20HjHJ6iifk='} 2024-04-10T23:03:11.590 DEBUG:teuthology.nuke:shortname: smithi155 2024-04-10T23:03:11.590 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T23:03:11.636 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi155.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650178', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:14.657682', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOm4scpyhYI54CYzQf4cN3kW49KVlup3fx8iiyzf5hYqx+x08KqUz3hzGeMbixTtO0QGv1ufyEBHJzS/9k8lqw8='} 2024-04-10T23:03:11.728 INFO:teuthology.orchestra.console.smithi136:Power off 2024-04-10T23:03:11.728 DEBUG:teuthology.orchestra.console.smithi136:pexpect command: ipmitool -H smithi136.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-10T23:03:11.765 INFO:teuthology.orchestra.console.smithi155:Power off 2024-04-10T23:03:11.765 DEBUG:teuthology.orchestra.console.smithi155:pexpect command: ipmitool -H smithi155.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-10T23:03:11.779 DEBUG:teuthology.orchestra.console.smithi136:power off output: Chassis Power Control: Down/Off 2024-04-10T23:03:11.780 DEBUG:teuthology.orchestra.console.smithi136:pexpect command: ipmitool -H smithi136.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:03:11.830 DEBUG:teuthology.orchestra.console.smithi155:power off output: Chassis Power Control: Down/Off 2024-04-10T23:03:11.831 DEBUG:teuthology.orchestra.console.smithi155:pexpect command: ipmitool -H smithi155.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:03:11.839 DEBUG:teuthology.orchestra.console.smithi136:check power output: Chassis Power is on 2024-04-10T23:03:11.883 DEBUG:teuthology.orchestra.console.smithi155:check power output: Chassis Power is on 2024-04-10T23:03:15.841 DEBUG:teuthology.orchestra.console.smithi136:pexpect command: ipmitool -H smithi136.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:03:15.884 DEBUG:teuthology.orchestra.console.smithi155:pexpect command: ipmitool -H smithi155.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:03:15.951 DEBUG:teuthology.orchestra.console.smithi136:check power output: Chassis Power is on 2024-04-10T23:03:15.994 DEBUG:teuthology.orchestra.console.smithi155:check power output: Chassis Power is on 2024-04-10T23:03:19.956 DEBUG:teuthology.orchestra.console.smithi136:pexpect command: ipmitool -H smithi136.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:03:19.995 DEBUG:teuthology.orchestra.console.smithi155:pexpect command: ipmitool -H smithi155.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:03:20.067 DEBUG:teuthology.orchestra.console.smithi136:check power output: Chassis Power is off 2024-04-10T23:03:20.104 DEBUG:teuthology.orchestra.console.smithi155:check power output: Chassis Power is off 2024-04-10T23:03:20.168 INFO:teuthology.orchestra.console.smithi136:Power off completed 2024-04-10T23:03:20.206 INFO:teuthology.orchestra.console.smithi155:Power off completed 2024-04-10T23:03:20.312 INFO:teuthology.run:Summary data: description: upgrade:pacific-x/stress-split/{0-distro/centos_8.stream_container_tools 0-roles 1-start 2-first-half-tasks/readwrite 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} duration: 904.5389459133148 failure_reason: '['','', ''Bugzilla. In case of problems, contact the owner of this repository.'', ''Enabling a Copr repository. Please note that this repository is not part'', ''Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable'', ''Please do not file bug reports about these packages in Fedora'', ''The Fedora Project does not exercise any power over the contents of'', ''and packages are not held to any quality or security level.'', ''of the main distribution, and quality may vary.'', ''this repository beyond the rules outlined in the Copr FAQ at'']' owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=5dbbd5d1ed484f8ca6195d8f0b0e8ce3 status: dead success: false 2024-04-10T23:03:20.313 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-10T23:03:20.421 INFO:teuthology.run:DEAD