2024-04-10T22:22:07.225 INFO:root:teuthology version: 0.0.1.dev267+g6c63784 2024-04-10T22:22:07.225 DEBUG:teuthology.run:Teuthology command: teuthology --archive /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650167 --description upgrade:pacific-x/parallel/{0-random-distro$/{centos_8.stream_container_tools} 0-start 1-tasks mon_election/classic upgrade-sequence workload/{ec-rados-default rados_api rados_loadgenbig rbd_import_export test_rbd_api test_rbd_python}} --name teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi --verbose --owner scheduled_teuthology@teuthology -- /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650167/orig.config.yaml 2024-04-10T22:22:07.270 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-10T22:22:07.350 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650167 branch: reef description: upgrade:pacific-x/parallel/{0-random-distro$/{centos_8.stream_container_tools} 0-start 1-tasks mon_election/classic upgrade-sequence workload/{ec-rados-default rados_api rados_loadgenbig rbd_import_export test_rbd_api test_rbd_python}} email: ceph-qa@ceph.com first_in_suite: false job_id: '7650167' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run run randomized correctness test for rados operations on an erasure-coded pool ' - desc: 'object class functional tests ' - desc: 'generate read/write load with rados objects ranging from 1MB to 25MB ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'librbd python api tests ' name: teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 8.stream overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: extra_system_packages: - python3-pytest flavor: default sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: reef sha1: 4f70a9a90881107996c8b7a3e89ba8b32624713c owner: scheduled_teuthology@teuthology priority: 830 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - mds.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mds.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b seed: 4604 sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 sleep_before_teardown: 0 subset: 0/1 suite: upgrade:pacific-x suite_branch: reef suite_path: /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 4f70a9a90881107996c8b7a3e89ba8b32624713c targets: smithi069.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJ7hDyuULrvi8kxi/ACWWPuMSR+jNdXlIDUowsd5lko7AhAHUl7XX5HajlZtwqfivNjUCX3KyQrehQCwovso6zg= smithi189.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPK9oW2fSYCzNlAQ0gb/x7Rf88zFppVpUqeSKGHRFjSF441RZpWmsVLQ3ELYv5lEM/ZDll7w9JQZLodjoyCHtPI= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - install: branch: pacific exclude_packages: - ceph-volume - print: '**** done install task...' - print: '**** done start installing pacific cephadm ...' - cephadm: cephadm_branch: pacific cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:pacific - print: '**** done end installing pacific cephadm ...' - print: '**** done start cephadm.shell ceph config set mgr...' - cephadm.shell: mon.a: - ceph config set mgr mgr/cephadm/use_repo_digest true --force - print: '**** done cephadm.shell ceph config set mgr...' - print: '**** done start telemetry pacific...' - workunit: clients: client.0: - test_telemetry_pacific.sh - print: '**** done end telemetry pacific...' - print: '**** done start parallel' - parallel: - workload - upgrade-sequence - print: '**** done end parallel' - print: '**** done start telemetry x...' - workunit: clients: client.0: - test_telemetry_pacific_x.sh - print: '**** done end telemetry x...' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 6c637841c215537a4502385240412f1966e0faab timestamp: 2024-04-10_01:08:08 tube: smithi upgrade-sequence: sequential: - print: '**** done start upgrade, wait' - cephadm.shell: env: - sha1 mon.a: - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - print: '**** done end upgrade, wait...' user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 workload: full_sequential: - print: '**** done start ec-rados-default.yaml' - rados: clients: - client.0 ec_pool: true objects: 50 op_weights: append: 100 copy_from: 50 delete: 50 read: 100 rmattr: 25 rollback: 50 setattr: 25 snap_create: 50 snap_remove: 50 write: 0 ops: 4000 write_append_excl: false - print: '**** done end ec-rados-default.yaml' - print: '**** done start rados_api.yaml' - workunit: branch: pacific clients: client.0: - cls - print: '**** done end rados_api.yaml' - print: '**** done start rados_loadgenbig.yaml' - workunit: branch: pacific clients: client.0: - rados/load-gen-big.sh - print: '**** done end rados_loadgenbig.yaml' - print: '**** done start rbd_import_export.yaml' - workunit: branch: pacific clients: client.1: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done end rbd_import_export.yaml' - print: '**** done start test_rbd_api.yaml' - workunit: branch: pacific clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done end test_rbd_api.yaml' - print: '**** done start test_rbd_python.yaml' - workunit: branch: pacific clients: client.0: - rbd/test_librbd_python.sh env: RBD_FEATURES: '61' - print: '**** done end test_rbd_python.yaml' 2024-04-10T22:22:07.351 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa; will attempt to use it 2024-04-10T22:22:07.456 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa/tasks 2024-04-10T22:22:07.456 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-10T22:22:07.458 INFO:teuthology.task.internal:Checking packages... 2024-04-10T22:22:07.478 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'd540ebaca6b131a1dd560e7f69e024b133bbaa42' 2024-04-10T22:22:07.478 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-10T22:22:07.478 INFO:teuthology.packaging:ref: None 2024-04-10T22:22:07.478 INFO:teuthology.packaging:tag: None 2024-04-10T22:22:07.478 INFO:teuthology.packaging:branch: reef 2024-04-10T22:22:07.478 INFO:teuthology.packaging:sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 2024-04-10T22:22:07.478 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef 2024-04-10T22:22:08.261 INFO:teuthology.task.internal:Found packages for ceph version 18.2.2-1189.gc96c8d47 2024-04-10T22:22:08.263 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-10T22:22:08.271 INFO:teuthology.task.internal:no buildpackages task found 2024-04-10T22:22:08.271 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-10T22:22:08.281 INFO:teuthology.task.internal:Saving configuration 2024-04-10T22:22:08.299 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-10T22:22:08.308 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T22:22:08.334 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi069.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650167', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:15:47.999610', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJ7hDyuULrvi8kxi/ACWWPuMSR+jNdXlIDUowsd5lko7AhAHUl7XX5HajlZtwqfivNjUCX3KyQrehQCwovso6zg='} 2024-04-10T22:22:08.358 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi189.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650167', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:15:48.001498', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPK9oW2fSYCzNlAQ0gb/x7Rf88zFppVpUqeSKGHRFjSF441RZpWmsVLQ3ELYv5lEM/ZDll7w9JQZLodjoyCHtPI='} 2024-04-10T22:22:08.358 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-10T22:22:08.365 INFO:teuthology.task.internal:roles: ubuntu@smithi069.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'mds.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-04-10T22:22:08.366 INFO:teuthology.task.internal:roles: ubuntu@smithi189.front.sepia.ceph.com - ['mon.b', 'mds.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-04-10T22:22:08.366 INFO:teuthology.run_tasks:Running task console_log... 2024-04-10T22:22:08.440 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f69650da0d0>, signals=[15]) 2024-04-10T22:22:08.440 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-10T22:22:08.448 INFO:teuthology.task.internal:Opening connections... 2024-04-10T22:22:08.449 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi069.front.sepia.ceph.com 2024-04-10T22:22:08.451 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi069.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:22:08.527 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi189.front.sepia.ceph.com 2024-04-10T22:22:08.528 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi189.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:22:08.599 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-10T22:22:08.608 DEBUG:teuthology.orchestra.run.smithi069:> uname -m 2024-04-10T22:22:08.630 INFO:teuthology.orchestra.run.smithi069.stdout:x86_64 2024-04-10T22:22:08.631 DEBUG:teuthology.orchestra.run.smithi069:> cat /etc/os-release 2024-04-10T22:22:08.691 INFO:teuthology.orchestra.run.smithi069.stdout:NAME="CentOS Stream" 2024-04-10T22:22:08.691 INFO:teuthology.orchestra.run.smithi069.stdout:VERSION="8" 2024-04-10T22:22:08.691 INFO:teuthology.orchestra.run.smithi069.stdout:ID="centos" 2024-04-10T22:22:08.691 INFO:teuthology.orchestra.run.smithi069.stdout:ID_LIKE="rhel fedora" 2024-04-10T22:22:08.692 INFO:teuthology.orchestra.run.smithi069.stdout:VERSION_ID="8" 2024-04-10T22:22:08.692 INFO:teuthology.orchestra.run.smithi069.stdout:PLATFORM_ID="platform:el8" 2024-04-10T22:22:08.692 INFO:teuthology.orchestra.run.smithi069.stdout:PRETTY_NAME="CentOS Stream 8" 2024-04-10T22:22:08.692 INFO:teuthology.orchestra.run.smithi069.stdout:ANSI_COLOR="0;31" 2024-04-10T22:22:08.692 INFO:teuthology.orchestra.run.smithi069.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2024-04-10T22:22:08.692 INFO:teuthology.orchestra.run.smithi069.stdout:HOME_URL="https://centos.org/" 2024-04-10T22:22:08.692 INFO:teuthology.orchestra.run.smithi069.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-10T22:22:08.692 INFO:teuthology.orchestra.run.smithi069.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2024-04-10T22:22:08.692 INFO:teuthology.orchestra.run.smithi069.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-10T22:22:08.693 INFO:teuthology.lock.ops:Updating smithi069.front.sepia.ceph.com on lock server 2024-04-10T22:22:08.714 DEBUG:teuthology.orchestra.run.smithi189:> uname -m 2024-04-10T22:22:08.736 INFO:teuthology.orchestra.run.smithi189.stdout:x86_64 2024-04-10T22:22:08.737 DEBUG:teuthology.orchestra.run.smithi189:> cat /etc/os-release 2024-04-10T22:22:08.798 INFO:teuthology.orchestra.run.smithi189.stdout:NAME="CentOS Stream" 2024-04-10T22:22:08.798 INFO:teuthology.orchestra.run.smithi189.stdout:VERSION="8" 2024-04-10T22:22:08.798 INFO:teuthology.orchestra.run.smithi189.stdout:ID="centos" 2024-04-10T22:22:08.798 INFO:teuthology.orchestra.run.smithi189.stdout:ID_LIKE="rhel fedora" 2024-04-10T22:22:08.798 INFO:teuthology.orchestra.run.smithi189.stdout:VERSION_ID="8" 2024-04-10T22:22:08.798 INFO:teuthology.orchestra.run.smithi189.stdout:PLATFORM_ID="platform:el8" 2024-04-10T22:22:08.798 INFO:teuthology.orchestra.run.smithi189.stdout:PRETTY_NAME="CentOS Stream 8" 2024-04-10T22:22:08.798 INFO:teuthology.orchestra.run.smithi189.stdout:ANSI_COLOR="0;31" 2024-04-10T22:22:08.798 INFO:teuthology.orchestra.run.smithi189.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2024-04-10T22:22:08.798 INFO:teuthology.orchestra.run.smithi189.stdout:HOME_URL="https://centos.org/" 2024-04-10T22:22:08.798 INFO:teuthology.orchestra.run.smithi189.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-10T22:22:08.798 INFO:teuthology.orchestra.run.smithi189.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2024-04-10T22:22:08.798 INFO:teuthology.orchestra.run.smithi189.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-10T22:22:08.801 INFO:teuthology.lock.ops:Updating smithi189.front.sepia.ceph.com on lock server 2024-04-10T22:22:08.822 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-10T22:22:08.833 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-10T22:22:08.839 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-10T22:22:08.839 DEBUG:teuthology.orchestra.run.smithi069:> test '!' -e /home/ubuntu/cephtest 2024-04-10T22:22:08.842 DEBUG:teuthology.orchestra.run.smithi189:> test '!' -e /home/ubuntu/cephtest 2024-04-10T22:22:08.862 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-10T22:22:08.870 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-10T22:22:08.870 DEBUG:teuthology.orchestra.run.smithi069:> test -z $(ls -A /var/lib/ceph) 2024-04-10T22:22:08.903 DEBUG:teuthology.orchestra.run.smithi189:> test -z $(ls -A /var/lib/ceph) 2024-04-10T22:22:08.935 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-10T22:22:08.988 INFO:teuthology.run_tasks:Running task kernel... 2024-04-10T22:22:09.002 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-04-10T22:22:09.002 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-04-10T22:22:09.002 DEBUG:teuthology.orchestra.run.smithi069:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-10T22:22:09.003 DEBUG:teuthology.orchestra.run.smithi189:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-10T22:22:09.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:22:09.023 DEBUG:teuthology.orchestra.run.smithi189:> uname -r 2024-04-10T22:22:09.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:22:09.024 DEBUG:teuthology.orchestra.run.smithi069:> uname -r 2024-04-10T22:22:09.083 INFO:teuthology.orchestra.run.smithi189.stdout:4.18.0-489.el8.x86_64 2024-04-10T22:22:09.083 INFO:teuthology.task.kernel:Running kernel on smithi189: 4.18.0-489.el8.x86_64 2024-04-10T22:22:09.083 DEBUG:teuthology.orchestra.run.smithi189:> sudo yum install -y kernel 2024-04-10T22:22:09.084 INFO:teuthology.orchestra.run.smithi069.stdout:4.18.0-489.el8.x86_64 2024-04-10T22:22:09.085 INFO:teuthology.task.kernel:Running kernel on smithi069: 4.18.0-489.el8.x86_64 2024-04-10T22:22:09.085 DEBUG:teuthology.orchestra.run.smithi069:> sudo yum install -y kernel 2024-04-10T22:22:13.513 INFO:teuthology.orchestra.run.smithi189.stdout:CentOS Stream 8 - AppStream 15 MB/s | 28 MB 00:01 2024-04-10T22:22:13.724 INFO:teuthology.orchestra.run.smithi069.stdout:CentOS Stream 8 - AppStream 13 MB/s | 28 MB 00:02 2024-04-10T22:22:20.263 INFO:teuthology.orchestra.run.smithi189.stdout:CentOS Stream 8 - BaseOS 5.8 MB/s | 10 MB 00:01 2024-04-10T22:24:32.303 INFO:teuthology.orchestra.run.smithi069.stdout:CentOS Stream 8 - BaseOS 79 kB/s | 10 MB 02:13 2024-04-10T22:24:49.836 INFO:teuthology.orchestra.run.smithi189.stdout:CentOS Stream 8 - Extras 123 B/s | 18 kB 02:27 2024-04-10T22:24:50.261 INFO:teuthology.orchestra.run.smithi189.stdout:CentOS Stream 8 - Extras common packages 25 kB/s | 7.7 kB 00:00 2024-04-10T22:24:52.585 INFO:teuthology.orchestra.run.smithi069.stdout:CentOS Stream 8 - Extras 976 B/s | 18 kB 00:18 2024-04-10T22:24:53.035 INFO:teuthology.orchestra.run.smithi069.stdout:CentOS Stream 8 - Extras common packages 23 kB/s | 7.7 kB 00:00 2024-04-10T22:25:30.614 INFO:teuthology.orchestra.run.smithi069.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:37 2024-04-10T22:25:30.614 INFO:teuthology.orchestra.run.smithi069.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:25:30.615 INFO:teuthology.orchestra.run.smithi069.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:25:30.615 INFO:teuthology.orchestra.run.smithi069.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:25:30.639 INFO:teuthology.orchestra.run.smithi189.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:40 2024-04-10T22:25:30.640 INFO:teuthology.orchestra.run.smithi189.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:25:30.640 INFO:teuthology.orchestra.run.smithi189.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:25:30.640 INFO:teuthology.orchestra.run.smithi189.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:25:32.035 INFO:teuthology.orchestra.run.smithi189.stdout:Extra Packages for Enterprise Linux 12 MB/s | 16 MB 00:01 2024-04-10T22:25:33.274 INFO:teuthology.orchestra.run.smithi069.stdout:Extra Packages for Enterprise Linux 6.1 MB/s | 16 MB 00:02 2024-04-10T22:25:35.841 INFO:teuthology.orchestra.run.smithi189.stdout:lab-extras 363 kB/s | 24 kB 00:00 2024-04-10T22:25:35.982 INFO:teuthology.orchestra.run.smithi189.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:25:37.065 INFO:teuthology.orchestra.run.smithi069.stdout:lab-extras 288 kB/s | 24 kB 00:00 2024-04-10T22:25:37.210 INFO:teuthology.orchestra.run.smithi069.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:25:38.754 INFO:teuthology.orchestra.run.smithi189.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2024-04-10T22:25:38.754 INFO:teuthology.orchestra.run.smithi189.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2024-04-10T22:25:38.863 INFO:teuthology.orchestra.run.smithi189.stdout:Dependencies resolved. 2024-04-10T22:25:38.863 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-04-10T22:25:38.864 INFO:teuthology.orchestra.run.smithi189.stdout: Package Architecture Version Repository Size 2024-04-10T22:25:38.864 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-04-10T22:25:38.864 INFO:teuthology.orchestra.run.smithi189.stdout:Installing: 2024-04-10T22:25:38.864 INFO:teuthology.orchestra.run.smithi189.stdout: kernel x86_64 4.18.0-552.el8 baseos 10 M 2024-04-10T22:25:38.864 INFO:teuthology.orchestra.run.smithi189.stdout:Installing dependencies: 2024-04-10T22:25:38.864 INFO:teuthology.orchestra.run.smithi189.stdout: kernel-core x86_64 4.18.0-552.el8 baseos 44 M 2024-04-10T22:25:38.864 INFO:teuthology.orchestra.run.smithi189.stdout: kernel-modules x86_64 4.18.0-552.el8 baseos 36 M 2024-04-10T22:25:38.864 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-04-10T22:25:38.864 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction Summary 2024-04-10T22:25:38.864 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-04-10T22:25:38.864 INFO:teuthology.orchestra.run.smithi189.stdout:Install 3 Packages 2024-04-10T22:25:38.864 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-04-10T22:25:38.865 INFO:teuthology.orchestra.run.smithi189.stdout:Total download size: 90 M 2024-04-10T22:25:38.865 INFO:teuthology.orchestra.run.smithi189.stdout:Installed size: 96 M 2024-04-10T22:25:38.865 INFO:teuthology.orchestra.run.smithi189.stdout:Downloading Packages: 2024-04-10T22:25:39.859 INFO:teuthology.orchestra.run.smithi189.stdout:(1/3): kernel-4.18.0-552.el8.x86_64.rpm 13 MB/s | 10 MB 00:00 2024-04-10T22:25:40.029 INFO:teuthology.orchestra.run.smithi069.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2024-04-10T22:25:40.029 INFO:teuthology.orchestra.run.smithi069.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2024-04-10T22:25:40.149 INFO:teuthology.orchestra.run.smithi069.stdout:Dependencies resolved. 2024-04-10T22:25:40.150 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2024-04-10T22:25:40.150 INFO:teuthology.orchestra.run.smithi069.stdout: Package Architecture Version Repository Size 2024-04-10T22:25:40.150 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2024-04-10T22:25:40.150 INFO:teuthology.orchestra.run.smithi069.stdout:Installing: 2024-04-10T22:25:40.150 INFO:teuthology.orchestra.run.smithi069.stdout: kernel x86_64 4.18.0-552.el8 baseos 10 M 2024-04-10T22:25:40.150 INFO:teuthology.orchestra.run.smithi069.stdout:Installing dependencies: 2024-04-10T22:25:40.150 INFO:teuthology.orchestra.run.smithi069.stdout: kernel-core x86_64 4.18.0-552.el8 baseos 44 M 2024-04-10T22:25:40.150 INFO:teuthology.orchestra.run.smithi069.stdout: kernel-modules x86_64 4.18.0-552.el8 baseos 36 M 2024-04-10T22:25:40.150 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-04-10T22:25:40.150 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction Summary 2024-04-10T22:25:40.150 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2024-04-10T22:25:40.150 INFO:teuthology.orchestra.run.smithi069.stdout:Install 3 Packages 2024-04-10T22:25:40.150 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-04-10T22:25:40.151 INFO:teuthology.orchestra.run.smithi069.stdout:Total download size: 90 M 2024-04-10T22:25:40.151 INFO:teuthology.orchestra.run.smithi069.stdout:Installed size: 96 M 2024-04-10T22:25:40.151 INFO:teuthology.orchestra.run.smithi069.stdout:Downloading Packages: 2024-04-10T22:25:41.976 INFO:teuthology.orchestra.run.smithi189.stdout:(2/3): kernel-core-4.18.0-552.el8.x86_64.rpm 15 MB/s | 44 MB 00:02 2024-04-10T22:25:46.642 INFO:teuthology.orchestra.run.smithi189.stdout:(3/3): kernel-modules-4.18.0-552.el8.x86_64.rpm 4.8 MB/s | 36 MB 00:07 2024-04-10T22:25:46.642 INFO:teuthology.orchestra.run.smithi189.stdout:-------------------------------------------------------------------------------- 2024-04-10T22:25:46.642 INFO:teuthology.orchestra.run.smithi189.stdout:Total 12 MB/s | 90 MB 00:07 2024-04-10T22:25:47.332 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction check 2024-04-10T22:25:47.743 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction check succeeded. 2024-04-10T22:25:47.744 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction test 2024-04-10T22:25:49.007 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction test succeeded. 2024-04-10T22:25:49.034 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction 2024-04-10T22:25:51.830 INFO:teuthology.orchestra.run.smithi189.stdout: Preparing : 1/1 2024-04-10T22:25:52.366 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:25:55.929 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:25:56.298 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:25:59.741 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:26:00.031 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : kernel-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:26:21.849 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:26:21.874 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:26:22.228 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: kernel-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:26:22.228 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : kernel-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:26:22.228 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : kernel-core-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:26:22.707 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:26:22.707 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-04-10T22:26:22.707 INFO:teuthology.orchestra.run.smithi189.stdout:Installed: 2024-04-10T22:26:22.707 INFO:teuthology.orchestra.run.smithi189.stdout: kernel-4.18.0-552.el8.x86_64 kernel-core-4.18.0-552.el8.x86_64 2024-04-10T22:26:22.707 INFO:teuthology.orchestra.run.smithi189.stdout: kernel-modules-4.18.0-552.el8.x86_64 2024-04-10T22:26:22.708 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-04-10T22:26:22.708 INFO:teuthology.orchestra.run.smithi189.stdout:Complete! 2024-04-10T22:26:22.883 DEBUG:teuthology.orchestra.run.smithi189:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:26:23.102 INFO:teuthology.orchestra.run.smithi189.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-10T22:26:23.103 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-10T22:26:23.103 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-552.el8.x86_64 2024-04-10T22:26:23.103 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-10T22:26:23.103 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-10T22:26:23.103 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-04-10T22:26:23.103 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi189.front.sepia.ceph.com, path=None, version=distro) 2024-04-10T22:26:23.103 DEBUG:teuthology.orchestra.run.smithi189:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:26:23.316 INFO:teuthology.orchestra.run.smithi189.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-10T22:26:23.316 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-10T22:26:23.317 DEBUG:teuthology.orchestra.run.smithi189:> sudo rpm -qi grub2-tools 2024-04-10T22:26:23.350 INFO:teuthology.orchestra.run.smithi189.stdout:Name : grub2-tools 2024-04-10T22:26:23.350 INFO:teuthology.orchestra.run.smithi189.stdout:Epoch : 1 2024-04-10T22:26:23.350 INFO:teuthology.orchestra.run.smithi189.stdout:Version : 2.02 2024-04-10T22:26:23.351 INFO:teuthology.orchestra.run.smithi189.stdout:Release : 129.el8 2024-04-10T22:26:23.351 INFO:teuthology.orchestra.run.smithi189.stdout:Architecture: x86_64 2024-04-10T22:26:23.351 INFO:teuthology.orchestra.run.smithi189.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2024-04-10T22:26:23.351 INFO:teuthology.orchestra.run.smithi189.stdout:Group : System Environment/Base 2024-04-10T22:26:23.351 INFO:teuthology.orchestra.run.smithi189.stdout:Size : 9265586 2024-04-10T22:26:23.351 INFO:teuthology.orchestra.run.smithi189.stdout:License : GPLv3+ 2024-04-10T22:26:23.351 INFO:teuthology.orchestra.run.smithi189.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2024-04-10T22:26:23.351 INFO:teuthology.orchestra.run.smithi189.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2024-04-10T22:26:23.351 INFO:teuthology.orchestra.run.smithi189.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2024-04-10T22:26:23.351 INFO:teuthology.orchestra.run.smithi189.stdout:Build Host : kbuilder.bsys.centos.org 2024-04-10T22:26:23.351 INFO:teuthology.orchestra.run.smithi189.stdout:Relocations : (not relocatable) 2024-04-10T22:26:23.351 INFO:teuthology.orchestra.run.smithi189.stdout:Packager : CentOS BuildSystem 2024-04-10T22:26:23.351 INFO:teuthology.orchestra.run.smithi189.stdout:Vendor : CentOS 2024-04-10T22:26:23.352 INFO:teuthology.orchestra.run.smithi189.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-10T22:26:23.352 INFO:teuthology.orchestra.run.smithi189.stdout:Summary : Support tools for GRUB. 2024-04-10T22:26:23.352 INFO:teuthology.orchestra.run.smithi189.stdout:Description : 2024-04-10T22:26:23.352 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-04-10T22:26:23.352 INFO:teuthology.orchestra.run.smithi189.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-10T22:26:23.352 INFO:teuthology.orchestra.run.smithi189.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-10T22:26:23.352 INFO:teuthology.orchestra.run.smithi189.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-10T22:26:23.352 INFO:teuthology.orchestra.run.smithi189.stdout:hardware devices. 2024-04-10T22:26:23.352 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-04-10T22:26:23.352 INFO:teuthology.orchestra.run.smithi189.stdout:This subpackage provides tools for support of all platforms. 2024-04-10T22:26:23.353 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-10T22:26:23.353 INFO:teuthology.task.kernel:Updating grub on smithi189 to boot 4.18.0-552.el8.x86_64 2024-04-10T22:26:23.353 DEBUG:teuthology.orchestra.run.smithi189:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-10T22:26:25.060 INFO:teuthology.orchestra.run.smithi189.stderr:Generating grub configuration file ... 2024-04-10T22:26:26.895 INFO:teuthology.orchestra.run.smithi189.stderr:done 2024-04-10T22:26:26.896 DEBUG:teuthology.orchestra.run.smithi189:> mktemp 2024-04-10T22:26:26.926 INFO:teuthology.orchestra.run.smithi189.stdout:/tmp/tmp.D8NgmzG4GU 2024-04-10T22:26:26.927 DEBUG:teuthology.orchestra.run.smithi189:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.D8NgmzG4GU 2024-04-10T22:26:27.000 DEBUG:teuthology.orchestra.run.smithi189:> sudo chmod 0666 /tmp/tmp.D8NgmzG4GU 2024-04-10T22:26:27.157 DEBUG:teuthology.orchestra.remote:smithi189:/tmp/tmp.D8NgmzG4GU is 6KB 2024-04-10T22:26:27.205 DEBUG:teuthology.orchestra.run.smithi189:> rm -fr /tmp/tmp.D8NgmzG4GU 2024-04-10T22:26:27.222 DEBUG:teuthology.orchestra.run.smithi189:> sudo /bin/ls /boot/loader/entries || true 2024-04-10T22:26:27.295 INFO:teuthology.orchestra.run.smithi189.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2024-04-10T22:26:27.296 INFO:teuthology.orchestra.run.smithi189.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2024-04-10T22:26:27.296 INFO:teuthology.orchestra.run.smithi189.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2024-04-10T22:26:27.296 INFO:teuthology.orchestra.run.smithi189.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64.conf 2024-04-10T22:26:27.297 DEBUG:teuthology.orchestra.run.smithi189:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64 2024-04-10T22:26:27.461 DEBUG:teuthology.orchestra.run.smithi189:> sudo shutdown -r now 2024-04-10T22:26:57.464 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-04-10T22:26:57.465 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi189.front.sepia.ceph.com' 2024-04-10T22:26:57.466 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi189.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:27:31.999 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.189 2024-04-10T22:27:41.000 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi189.front.sepia.ceph.com' 2024-04-10T22:27:41.001 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi189.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:27:59.394 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.189 2024-04-10T22:27:59.565 INFO:teuthology.orchestra.run.smithi069.stdout:(1/3): kernel-4.18.0-552.el8.x86_64.rpm 1.4 MB/s | 10 MB 00:07 2024-04-10T22:28:06.322 INFO:teuthology.orchestra.run.smithi069.stdout:(2/3): kernel-core-4.18.0-552.el8.x86_64.rpm 3.1 MB/s | 44 MB 00:14 2024-04-10T22:28:08.497 INFO:teuthology.orchestra.run.smithi069.stdout:(3/3): kernel-modules-4.18.0-552.el8.x86_64.rpm 2.2 MB/s | 36 MB 00:16 2024-04-10T22:28:08.498 INFO:teuthology.orchestra.run.smithi069.stdout:-------------------------------------------------------------------------------- 2024-04-10T22:28:08.498 INFO:teuthology.orchestra.run.smithi069.stdout:Total 624 kB/s | 90 MB 02:28 2024-04-10T22:28:09.177 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction check 2024-04-10T22:28:09.606 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction check succeeded. 2024-04-10T22:28:09.606 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction test 2024-04-10T22:28:10.850 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction test succeeded. 2024-04-10T22:28:10.887 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction 2024-04-10T22:28:11.395 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi189.front.sepia.ceph.com' 2024-04-10T22:28:11.396 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi189.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:28:13.063 DEBUG:teuthology.orchestra.run.smithi189:> true 2024-04-10T22:28:13.349 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi189.front.sepia.ceph.com' 2024-04-10T22:28:13.349 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "4.18.0-552.el8.x86_64"... 2024-04-10T22:28:13.349 DEBUG:teuthology.orchestra.run.smithi189:> uname -r 2024-04-10T22:28:13.368 INFO:teuthology.orchestra.run.smithi189.stdout:4.18.0-552.el8.x86_64 2024-04-10T22:28:13.369 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-552.el8.x86_64 vs 4.18.0-552.el8.x86_64 2024-04-10T22:28:13.369 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-10T22:28:13.369 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-10T22:28:13.677 INFO:teuthology.orchestra.run.smithi069.stdout: Preparing : 1/1 2024-04-10T22:28:14.264 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:28:14.370 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-04-10T22:28:14.370 DEBUG:teuthology.orchestra.run.smithi189:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-10T22:28:14.426 INFO:teuthology.orchestra.run.smithi189.stdout:ttyS1 2024-04-10T22:28:14.458 DEBUG:teuthology.parallel:result is None 2024-04-10T22:28:17.814 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:28:18.222 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:28:21.660 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:28:21.955 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : kernel-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:28:43.898 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:28:43.922 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:28:44.376 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: kernel-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:28:44.376 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : kernel-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:28:44.376 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : kernel-core-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:28:44.896 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:28:44.896 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-04-10T22:28:44.896 INFO:teuthology.orchestra.run.smithi069.stdout:Installed: 2024-04-10T22:28:44.896 INFO:teuthology.orchestra.run.smithi069.stdout: kernel-4.18.0-552.el8.x86_64 kernel-core-4.18.0-552.el8.x86_64 2024-04-10T22:28:44.896 INFO:teuthology.orchestra.run.smithi069.stdout: kernel-modules-4.18.0-552.el8.x86_64 2024-04-10T22:28:44.896 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-04-10T22:28:44.896 INFO:teuthology.orchestra.run.smithi069.stdout:Complete! 2024-04-10T22:28:45.049 DEBUG:teuthology.orchestra.run.smithi069:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:28:45.264 INFO:teuthology.orchestra.run.smithi069.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-10T22:28:45.265 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-10T22:28:45.265 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-552.el8.x86_64 2024-04-10T22:28:45.265 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-10T22:28:45.265 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-10T22:28:45.265 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-04-10T22:28:45.265 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi069.front.sepia.ceph.com, path=None, version=distro) 2024-04-10T22:28:45.265 DEBUG:teuthology.orchestra.run.smithi069:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:28:45.475 INFO:teuthology.orchestra.run.smithi069.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-10T22:28:45.475 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-10T22:28:45.476 DEBUG:teuthology.orchestra.run.smithi069:> sudo rpm -qi grub2-tools 2024-04-10T22:28:45.509 INFO:teuthology.orchestra.run.smithi069.stdout:Name : grub2-tools 2024-04-10T22:28:45.509 INFO:teuthology.orchestra.run.smithi069.stdout:Epoch : 1 2024-04-10T22:28:45.509 INFO:teuthology.orchestra.run.smithi069.stdout:Version : 2.02 2024-04-10T22:28:45.509 INFO:teuthology.orchestra.run.smithi069.stdout:Release : 129.el8 2024-04-10T22:28:45.509 INFO:teuthology.orchestra.run.smithi069.stdout:Architecture: x86_64 2024-04-10T22:28:45.509 INFO:teuthology.orchestra.run.smithi069.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2024-04-10T22:28:45.509 INFO:teuthology.orchestra.run.smithi069.stdout:Group : System Environment/Base 2024-04-10T22:28:45.509 INFO:teuthology.orchestra.run.smithi069.stdout:Size : 9265586 2024-04-10T22:28:45.509 INFO:teuthology.orchestra.run.smithi069.stdout:License : GPLv3+ 2024-04-10T22:28:45.509 INFO:teuthology.orchestra.run.smithi069.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2024-04-10T22:28:45.509 INFO:teuthology.orchestra.run.smithi069.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2024-04-10T22:28:45.509 INFO:teuthology.orchestra.run.smithi069.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2024-04-10T22:28:45.510 INFO:teuthology.orchestra.run.smithi069.stdout:Build Host : kbuilder.bsys.centos.org 2024-04-10T22:28:45.510 INFO:teuthology.orchestra.run.smithi069.stdout:Relocations : (not relocatable) 2024-04-10T22:28:45.510 INFO:teuthology.orchestra.run.smithi069.stdout:Packager : CentOS BuildSystem 2024-04-10T22:28:45.510 INFO:teuthology.orchestra.run.smithi069.stdout:Vendor : CentOS 2024-04-10T22:28:45.510 INFO:teuthology.orchestra.run.smithi069.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-10T22:28:45.510 INFO:teuthology.orchestra.run.smithi069.stdout:Summary : Support tools for GRUB. 2024-04-10T22:28:45.510 INFO:teuthology.orchestra.run.smithi069.stdout:Description : 2024-04-10T22:28:45.510 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-04-10T22:28:45.510 INFO:teuthology.orchestra.run.smithi069.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-10T22:28:45.510 INFO:teuthology.orchestra.run.smithi069.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-10T22:28:45.510 INFO:teuthology.orchestra.run.smithi069.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-10T22:28:45.510 INFO:teuthology.orchestra.run.smithi069.stdout:hardware devices. 2024-04-10T22:28:45.510 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-04-10T22:28:45.511 INFO:teuthology.orchestra.run.smithi069.stdout:This subpackage provides tools for support of all platforms. 2024-04-10T22:28:45.511 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-10T22:28:45.511 INFO:teuthology.task.kernel:Updating grub on smithi069 to boot 4.18.0-552.el8.x86_64 2024-04-10T22:28:45.512 DEBUG:teuthology.orchestra.run.smithi069:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-10T22:28:46.946 INFO:teuthology.orchestra.run.smithi069.stderr:Generating grub configuration file ... 2024-04-10T22:28:49.195 INFO:teuthology.orchestra.run.smithi069.stderr:done 2024-04-10T22:28:49.198 DEBUG:teuthology.orchestra.run.smithi069:> mktemp 2024-04-10T22:28:49.229 INFO:teuthology.orchestra.run.smithi069.stdout:/tmp/tmp.kZzhR4wgzd 2024-04-10T22:28:49.229 DEBUG:teuthology.orchestra.run.smithi069:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.kZzhR4wgzd 2024-04-10T22:28:49.313 DEBUG:teuthology.orchestra.run.smithi069:> sudo chmod 0666 /tmp/tmp.kZzhR4wgzd 2024-04-10T22:28:49.454 DEBUG:teuthology.orchestra.remote:smithi069:/tmp/tmp.kZzhR4wgzd is 6KB 2024-04-10T22:28:49.503 DEBUG:teuthology.orchestra.run.smithi069:> rm -fr /tmp/tmp.kZzhR4wgzd 2024-04-10T22:28:49.520 DEBUG:teuthology.orchestra.run.smithi069:> sudo /bin/ls /boot/loader/entries || true 2024-04-10T22:28:49.596 INFO:teuthology.orchestra.run.smithi069.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2024-04-10T22:28:49.597 INFO:teuthology.orchestra.run.smithi069.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2024-04-10T22:28:49.597 INFO:teuthology.orchestra.run.smithi069.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2024-04-10T22:28:49.597 INFO:teuthology.orchestra.run.smithi069.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64.conf 2024-04-10T22:28:49.598 DEBUG:teuthology.orchestra.run.smithi069:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64 2024-04-10T22:28:49.799 DEBUG:teuthology.orchestra.run.smithi069:> sudo shutdown -r now 2024-04-10T22:29:19.802 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-04-10T22:29:19.803 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi069.front.sepia.ceph.com' 2024-04-10T22:29:19.804 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi069.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:29:38.206 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.69 2024-04-10T22:29:47.208 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi069.front.sepia.ceph.com' 2024-04-10T22:29:47.209 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi069.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:29:50.274 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.69 2024-04-10T22:30:02.275 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi069.front.sepia.ceph.com' 2024-04-10T22:30:02.276 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi069.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:30:20.707 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.69 2024-04-10T22:30:35.708 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi069.front.sepia.ceph.com' 2024-04-10T22:30:35.709 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi069.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:30:36.202 DEBUG:teuthology.orchestra.run.smithi069:> true 2024-04-10T22:30:37.270 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi069.front.sepia.ceph.com' 2024-04-10T22:30:37.270 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "4.18.0-552.el8.x86_64"... 2024-04-10T22:30:37.271 DEBUG:teuthology.orchestra.run.smithi069:> uname -r 2024-04-10T22:30:37.750 INFO:teuthology.orchestra.run.smithi069.stdout:4.18.0-552.el8.x86_64 2024-04-10T22:30:37.750 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-552.el8.x86_64 vs 4.18.0-552.el8.x86_64 2024-04-10T22:30:37.750 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-10T22:30:37.750 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-10T22:30:38.751 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-04-10T22:30:38.752 DEBUG:teuthology.orchestra.run.smithi069:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-10T22:30:38.808 INFO:teuthology.orchestra.run.smithi069.stdout:ttyS1 2024-04-10T22:30:38.920 DEBUG:teuthology.parallel:result is None 2024-04-10T22:30:38.921 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-10T22:30:38.929 INFO:teuthology.task.internal:Creating test directory... 2024-04-10T22:30:38.929 DEBUG:teuthology.orchestra.run.smithi069:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-10T22:30:38.931 DEBUG:teuthology.orchestra.run.smithi189:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-10T22:30:38.956 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-10T22:30:38.963 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-10T22:30:38.970 INFO:teuthology.task.internal:Creating archive directory... 2024-04-10T22:30:38.970 DEBUG:teuthology.orchestra.run.smithi069:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-10T22:30:38.992 DEBUG:teuthology.orchestra.run.smithi189:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-10T22:30:39.041 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-10T22:30:39.048 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-10T22:30:39.048 DEBUG:teuthology.orchestra.run.smithi069:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-10T22:30:39.064 DEBUG:teuthology.orchestra.run.smithi189:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-10T22:30:39.095 INFO:teuthology.orchestra.run.smithi069.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:30:39.105 INFO:teuthology.orchestra.run.smithi069.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:30:39.127 INFO:teuthology.orchestra.run.smithi189.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:30:39.138 INFO:teuthology.orchestra.run.smithi189.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:30:39.139 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-10T22:30:39.147 INFO:teuthology.task.internal:Configuring sudo... 2024-04-10T22:30:39.148 DEBUG:teuthology.orchestra.run.smithi069:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-10T22:30:39.153 DEBUG:teuthology.orchestra.run.smithi189:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-10T22:30:39.184 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-10T22:30:39.192 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-10T22:30:39.192 DEBUG:teuthology.orchestra.run.smithi069:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-10T22:30:39.223 DEBUG:teuthology.orchestra.run.smithi189:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-10T22:30:39.242 DEBUG:teuthology.orchestra.run.smithi069:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:30:39.329 DEBUG:teuthology.orchestra.run.smithi069:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:30:39.382 DEBUG:teuthology.orchestra.run.smithi069:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:30:39.470 DEBUG:teuthology.orchestra.run.smithi069:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:30:39.516 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2024-04-10T22:30:39.516 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-10T22:30:39.583 DEBUG:teuthology.orchestra.run.smithi189:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:30:39.632 DEBUG:teuthology.orchestra.run.smithi189:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:30:39.685 DEBUG:teuthology.orchestra.run.smithi189:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:30:39.770 DEBUG:teuthology.orchestra.run.smithi189:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:30:39.815 DEBUG:teuthology.orchestra.run.smithi189:> set -ex 2024-04-10T22:30:39.815 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-10T22:30:39.883 DEBUG:teuthology.orchestra.run.smithi069:> sudo service rsyslog restart 2024-04-10T22:30:39.885 DEBUG:teuthology.orchestra.run.smithi189:> sudo service rsyslog restart 2024-04-10T22:30:39.931 INFO:teuthology.orchestra.run.smithi069.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:30:39.993 INFO:teuthology.orchestra.run.smithi189.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:30:40.419 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-10T22:30:40.427 INFO:teuthology.task.internal:Starting timer... 2024-04-10T22:30:40.427 INFO:teuthology.run_tasks:Running task pcp... 2024-04-10T22:30:40.443 INFO:teuthology.run_tasks:Running task selinux... 2024-04-10T22:30:40.450 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-04-10T22:30:40.451 DEBUG:teuthology.orchestra.run.smithi069:> sudo service auditd rotate 2024-04-10T22:30:40.588 INFO:teuthology.orchestra.run.smithi069.stdout:Rotating logs: 2024-04-10T22:30:40.589 DEBUG:teuthology.orchestra.run.smithi189:> sudo service auditd rotate 2024-04-10T22:30:40.668 INFO:teuthology.orchestra.run.smithi189.stdout:Rotating logs: 2024-04-10T22:30:40.670 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-10T22:30:40.670 DEBUG:teuthology.orchestra.run.smithi069:> /usr/sbin/getenforce 2024-04-10T22:30:40.722 INFO:teuthology.orchestra.run.smithi069.stdout:Permissive 2024-04-10T22:30:40.722 DEBUG:teuthology.orchestra.run.smithi189:> /usr/sbin/getenforce 2024-04-10T22:30:40.746 INFO:teuthology.orchestra.run.smithi189.stdout:Permissive 2024-04-10T22:30:40.746 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi069.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi189.front.sepia.ceph.com': 'permissive'} 2024-04-10T22:30:40.746 DEBUG:teuthology.orchestra.run.smithi069:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:30:40.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:30:40.773 DEBUG:teuthology.orchestra.run.smithi189:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:30:40.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:30:40.813 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-10T22:30:40.813 DEBUG:teuthology.orchestra.run.smithi069:> sudo /usr/sbin/setenforce permissive 2024-04-10T22:30:40.841 DEBUG:teuthology.orchestra.run.smithi189:> sudo /usr/sbin/setenforce permissive 2024-04-10T22:30:40.880 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-10T22:30:40.890 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-04-10T22:30:40.895 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-04-10T22:30:41.034 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-10T22:30:41.049 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-10T22:30:41.051 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi069.front.sepia.ceph.com,smithi189.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-10T22:44:58.867 INFO:teuthology.task.ansible:Archiving ansible failure log at: /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650167/ansible_failures.yaml 2024-04-10T22:44:58.869 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/__init__.py", line 123, in __enter__ self.begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 508, in begin super(CephLab, self).begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 342, in begin self.execute_playbook() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 374, in execute_playbook self._handle_failure(command, status) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 402, in _handle_failure raise AnsibleFailedError(failures) teuthology.exceptions.AnsibleFailedError: [',', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at'] 2024-04-10T22:44:59.095 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=ef086d5d418f43a8a2ad0ab7263af684 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/__init__.py", line 123, in __enter__ self.begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 508, in begin super(CephLab, self).begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 342, in begin self.execute_playbook() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 374, in execute_playbook self._handle_failure(command, status) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 402, in _handle_failure raise AnsibleFailedError(failures) teuthology.exceptions.AnsibleFailedError: [',', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at'] 2024-04-10T22:44:59.098 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-10T22:44:59.108 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-10T22:44:59.108 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-10T22:44:59.116 DEBUG:teuthology.orchestra.run.smithi069:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-10T22:44:59.226 DEBUG:teuthology.orchestra.run.smithi189:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-10T22:44:59.294 DEBUG:teuthology.orchestra.run.smithi069:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:44:59.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:44:59.326 DEBUG:teuthology.orchestra.run.smithi189:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:44:59.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:44:59.357 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-10T22:44:59.365 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-10T22:44:59.375 INFO:teuthology.task.internal:Duration was 858.948519 seconds 2024-04-10T22:44:59.376 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-10T22:44:59.388 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-10T22:44:59.388 DEBUG:teuthology.orchestra.run.smithi069:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-10T22:44:59.390 DEBUG:teuthology.orchestra.run.smithi189:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-10T22:44:59.439 INFO:teuthology.orchestra.run.smithi069.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:44:59.446 INFO:teuthology.orchestra.run.smithi189.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:44:59.739 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-10T22:44:59.739 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi069.front.sepia.ceph.com 2024-04-10T22:44:59.739 DEBUG:teuthology.orchestra.run.smithi069:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-10T22:44:59.764 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi189.front.sepia.ceph.com 2024-04-10T22:44:59.765 DEBUG:teuthology.orchestra.run.smithi189:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-10T22:44:59.788 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-10T22:44:59.788 DEBUG:teuthology.orchestra.run.smithi069:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-10T22:44:59.807 DEBUG:teuthology.orchestra.run.smithi189:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-10T22:44:59.868 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-10T22:44:59.869 DEBUG:teuthology.orchestra.run.smithi069:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-10T22:44:59.891 DEBUG:teuthology.orchestra.run.smithi189:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-10T22:44:59.970 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-10T22:44:59.981 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-10T22:44:59.981 DEBUG:teuthology.orchestra.run.smithi069:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-10T22:45:00.008 DEBUG:teuthology.orchestra.run.smithi189:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-10T22:45:00.035 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-10T22:45:00.044 DEBUG:teuthology.orchestra.run.smithi069:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-10T22:45:00.051 DEBUG:teuthology.orchestra.run.smithi189:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-10T22:45:00.075 INFO:teuthology.orchestra.run.smithi069.stdout:kernel.core_pattern = core 2024-04-10T22:45:00.101 INFO:teuthology.orchestra.run.smithi189.stdout:kernel.core_pattern = core 2024-04-10T22:45:00.126 DEBUG:teuthology.orchestra.run.smithi069:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-10T22:45:00.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:45:00.165 DEBUG:teuthology.orchestra.run.smithi189:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-10T22:45:00.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:45:00.183 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-10T22:45:00.192 INFO:teuthology.task.internal:Transferring archived files... 2024-04-10T22:45:00.193 DEBUG:teuthology.misc:Transferring archived files from smithi069:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650167/remote/smithi069 2024-04-10T22:45:00.194 DEBUG:teuthology.orchestra.run.smithi069:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-10T22:45:00.251 DEBUG:teuthology.misc:Transferring archived files from smithi189:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650167/remote/smithi189 2024-04-10T22:45:00.252 DEBUG:teuthology.orchestra.run.smithi189:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-10T22:45:00.302 INFO:teuthology.task.internal:Removing archive directory... 2024-04-10T22:45:00.303 DEBUG:teuthology.orchestra.run.smithi069:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-10T22:45:00.305 DEBUG:teuthology.orchestra.run.smithi189:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-10T22:45:00.354 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-10T22:45:00.365 INFO:teuthology.task.internal:Not uploading archives. 2024-04-10T22:45:00.366 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-10T22:45:00.382 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-10T22:45:00.382 DEBUG:teuthology.orchestra.run.smithi069:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-10T22:45:00.384 DEBUG:teuthology.orchestra.run.smithi189:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-10T22:45:00.399 INFO:teuthology.orchestra.run.smithi069.stdout: 525355 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 10 22:45 /home/ubuntu/cephtest 2024-04-10T22:45:00.412 INFO:teuthology.orchestra.run.smithi189.stdout: 393224 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 10 22:45 /home/ubuntu/cephtest 2024-04-10T22:45:00.413 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-10T22:45:00.423 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-10T22:45:00.517 INFO:teuthology.nuke:Checking targets against current locks 2024-04-10T22:45:00.560 DEBUG:teuthology.nuke:shortname: smithi069 2024-04-10T22:45:00.560 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T22:45:00.616 DEBUG:teuthology.nuke:shortname: smithi189 2024-04-10T22:45:00.616 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T22:45:00.635 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi069.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650167', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:15:47.999610', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJ7hDyuULrvi8kxi/ACWWPuMSR+jNdXlIDUowsd5lko7AhAHUl7XX5HajlZtwqfivNjUCX3KyQrehQCwovso6zg='} 2024-04-10T22:45:00.674 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi189.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650167', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:15:48.001498', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPK9oW2fSYCzNlAQ0gb/x7Rf88zFppVpUqeSKGHRFjSF441RZpWmsVLQ3ELYv5lEM/ZDll7w9JQZLodjoyCHtPI='} 2024-04-10T22:45:00.760 INFO:teuthology.orchestra.console.smithi069:Power off 2024-04-10T22:45:00.760 DEBUG:teuthology.orchestra.console.smithi069:pexpect command: ipmitool -H smithi069.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-10T22:45:00.797 INFO:teuthology.orchestra.console.smithi189:Power off 2024-04-10T22:45:00.798 DEBUG:teuthology.orchestra.console.smithi189:pexpect command: ipmitool -H smithi189.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-10T22:45:00.819 DEBUG:teuthology.orchestra.console.smithi069:power off output: Chassis Power Control: Down/Off 2024-04-10T22:45:00.820 DEBUG:teuthology.orchestra.console.smithi069:pexpect command: ipmitool -H smithi069.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:45:00.857 DEBUG:teuthology.orchestra.console.smithi189:power off output: Chassis Power Control: Down/Off 2024-04-10T22:45:00.858 DEBUG:teuthology.orchestra.console.smithi189:pexpect command: ipmitool -H smithi189.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:45:00.866 DEBUG:teuthology.orchestra.console.smithi069:check power output: Chassis Power is on 2024-04-10T22:45:00.897 DEBUG:teuthology.orchestra.console.smithi189:check power output: Chassis Power is on 2024-04-10T22:45:04.868 DEBUG:teuthology.orchestra.console.smithi069:pexpect command: ipmitool -H smithi069.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:45:04.898 DEBUG:teuthology.orchestra.console.smithi189:pexpect command: ipmitool -H smithi189.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:45:04.978 DEBUG:teuthology.orchestra.console.smithi069:check power output: Chassis Power is on 2024-04-10T22:45:05.007 DEBUG:teuthology.orchestra.console.smithi189:check power output: Chassis Power is on 2024-04-10T22:45:08.982 DEBUG:teuthology.orchestra.console.smithi069:pexpect command: ipmitool -H smithi069.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:45:09.008 DEBUG:teuthology.orchestra.console.smithi189:pexpect command: ipmitool -H smithi189.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:45:09.092 DEBUG:teuthology.orchestra.console.smithi069:check power output: Chassis Power is off 2024-04-10T22:45:09.118 DEBUG:teuthology.orchestra.console.smithi189:check power output: Chassis Power is off 2024-04-10T22:45:09.193 INFO:teuthology.orchestra.console.smithi069:Power off completed 2024-04-10T22:45:09.219 INFO:teuthology.orchestra.console.smithi189:Power off completed 2024-04-10T22:45:09.325 INFO:teuthology.run:Summary data: description: upgrade:pacific-x/parallel/{0-random-distro$/{centos_8.stream_container_tools} 0-start 1-tasks mon_election/classic upgrade-sequence workload/{ec-rados-default rados_api rados_loadgenbig rbd_import_export test_rbd_api test_rbd_python}} duration: 858.9485194683075 failure_reason: '['','', ''Bugzilla. In case of problems, contact the owner of this repository.'', ''Enabling a Copr repository. Please note that this repository is not part'', ''Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable'', ''Please do not file bug reports about these packages in Fedora'', ''The Fedora Project does not exercise any power over the contents of'', ''and packages are not held to any quality or security level.'', ''of the main distribution, and quality may vary.'', ''this repository beyond the rules outlined in the Copr FAQ at'']' owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=ef086d5d418f43a8a2ad0ab7263af684 status: dead success: false 2024-04-10T22:45:09.326 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-10T22:45:09.456 INFO:teuthology.run:DEAD