2023-12-12T00:46:32.632 INFO:root:teuthology version: 0.0.1.dev216+g5f3731e 2023-12-12T00:46:32.632 DEBUG:teuthology.run:Teuthology command: teuthology --owner scheduled_yuriw@teuthology --verbose --archive /home/teuthworker/archive/yuriw-2023-12-11_23:27:14-rados-wip-yuri8-testing-2023-12-11-1101-distro-default-smithi/7487649 --name yuriw-2023-12-11_23:27:14-rados-wip-yuri8-testing-2023-12-11-1101-distro-default-smithi --description rados/upgrade/parallel/{0-random-distro$/{centos_8.stream_container_tools} 0-start 1-tasks mon_election/classic upgrade-sequence workload/{ec-rados-default rados_api rados_loadgenbig rbd_import_export test_rbd_api test_rbd_python}} -- /home/teuthworker/archive/yuriw-2023-12-11_23:27:14-rados-wip-yuri8-testing-2023-12-11-1101-distro-default-smithi/7487649/orig.config.yaml 2023-12-12T00:46:32.681 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-12-12T00:46:32.775 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/yuriw-2023-12-11_23:27:14-rados-wip-yuri8-testing-2023-12-11-1101-distro-default-smithi/7487649 branch: wip-yuri8-testing-2023-12-11-1101 description: rados/upgrade/parallel/{0-random-distro$/{centos_8.stream_container_tools} 0-start 1-tasks mon_election/classic upgrade-sequence workload/{ec-rados-default rados_api rados_loadgenbig rbd_import_export test_rbd_api test_rbd_python}} email: yweinste@redhat.com first_in_suite: false job_id: '7487649' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run run randomized correctness test for rados operations on an erasure-coded pool ' - desc: 'object class functional tests ' - desc: 'generate read/write load with rados objects ranging from 1MB to 25MB ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'librbd python api tests ' name: yuriw-2023-12-11_23:27:14-rados-wip-yuri8-testing-2023-12-11-1101-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 8.stream overrides: admin_socket: branch: wip-yuri8-testing-2023-12-11-1101 ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: 21b8309530d48c7266e1db5c3ccbe10963f99737 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: extra_system_packages: - python3-pytest flavor: default sha1: 21b8309530d48c7266e1db5c3ccbe10963f99737 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: wip-yuri8-testing-2023-12-11-1101 sha1: 21b8309530d48c7266e1db5c3ccbe10963f99737 owner: scheduled_yuriw@teuthology priority: 75 repo: https://github.com/ceph/ceph-ci.git roles: - - mon.a - mon.c - mgr.y - mds.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mds.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b seed: 3273 sha1: 21b8309530d48c7266e1db5c3ccbe10963f99737 sleep_before_teardown: 0 subset: 111/120000 suite: rados suite_branch: wip-yuri8-testing-2023-12-11-1101 suite_path: /home/teuthworker/src/github.com_ceph_ceph-c_21b8309530d48c7266e1db5c3ccbe10963f99737/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph-ci.git suite_sha1: 21b8309530d48c7266e1db5c3ccbe10963f99737 targets: smithi134.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNSwnXWJLVd/UmENlKG+RCgGnEAnwfxko3OHHXUu2ek5qu+LFa0GRhNbWUwK/TggSPq3XKdNfoiFGirPpVTk4is= smithi159.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOToXyQy09qfcRgE2GT6B8E54p/tvsTo+NPPQXjPwHjrxPr0sjFm17JZ2Q8fFGzY/A6lwkqoAL9RIDx1r4qExRU= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - install: branch: quincy exclude_packages: - ceph-volume - print: '**** done install task...' - print: '**** done start installing quincy cephadm ...' - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - print: '**** done end installing quincy cephadm ...' - print: '**** done start cephadm.shell ceph config set mgr...' - cephadm.shell: mon.a: - ceph config set mgr mgr/cephadm/use_repo_digest true --force - print: '**** done cephadm.shell ceph config set mgr...' - print: '**** done start telemetry quincy...' - workunit: clients: client.0: - test_telemetry_quincy.sh - print: '**** done end telemetry quincy...' - print: '**** done start parallel' - parallel: - workload - upgrade-sequence - print: '**** done end parallel' - print: '**** done start telemetry x...' - workunit: clients: client.0: - test_telemetry_quincy_x.sh - print: '**** done end telemetry x...' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 5f3731ee8689ef3b0ef7d571e4cc2cb968c78d73 timestamp: 2023-12-11_23:27:14 tube: smithi upgrade-sequence: sequential: - print: '**** done start upgrade, wait' - cephadm.shell: env: - sha1 mon.a: - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - print: '**** done end upgrade, wait...' user: yuriw verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.232157 workload: full_sequential: - print: '**** done start ec-rados-default.yaml' - rados: clients: - client.0 ec_pool: true objects: 50 op_weights: append: 100 copy_from: 50 delete: 50 read: 100 rmattr: 25 rollback: 50 setattr: 25 snap_create: 50 snap_remove: 50 write: 0 ops: 4000 write_append_excl: false - print: '**** done end ec-rados-default.yaml' - print: '**** done start rados_api.yaml' - workunit: branch: quincy clients: client.0: - cls - print: '**** done end rados_api.yaml' - print: '**** done start rados_loadgenbig.yaml' - workunit: branch: quincy clients: client.0: - rados/load-gen-big.sh - print: '**** done end rados_loadgenbig.yaml' - print: '**** done start rbd_import_export.yaml' - workunit: branch: quincy clients: client.1: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done end rbd_import_export.yaml' - print: '**** done start test_rbd_api.yaml' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done end test_rbd_api.yaml' - print: '**** done start test_rbd_python.yaml' - workunit: branch: quincy clients: client.0: - rbd/test_librbd_python.sh env: RBD_FEATURES: '61' - print: '**** done end test_rbd_python.yaml' 2023-12-12T00:46:32.775 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph-c_21b8309530d48c7266e1db5c3ccbe10963f99737/qa; will attempt to use it 2023-12-12T00:46:32.776 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph-c_21b8309530d48c7266e1db5c3ccbe10963f99737/qa/tasks 2023-12-12T00:46:32.776 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-12-12T00:46:32.778 INFO:teuthology.task.internal:Checking packages... 2023-12-12T00:46:32.806 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '21b8309530d48c7266e1db5c3ccbe10963f99737' 2023-12-12T00:46:32.806 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-12-12T00:46:32.806 INFO:teuthology.packaging:ref: None 2023-12-12T00:46:32.806 INFO:teuthology.packaging:tag: None 2023-12-12T00:46:32.806 INFO:teuthology.packaging:branch: wip-yuri8-testing-2023-12-11-1101 2023-12-12T00:46:32.807 INFO:teuthology.packaging:sha1: 21b8309530d48c7266e1db5c3ccbe10963f99737 2023-12-12T00:46:32.807 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=wip-yuri8-testing-2023-12-11-1101 2023-12-12T00:46:32.948 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-130.g21b83095 2023-12-12T00:46:32.949 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-12-12T00:46:32.956 INFO:teuthology.task.internal:no buildpackages task found 2023-12-12T00:46:32.956 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-12-12T00:46:32.971 INFO:teuthology.task.internal:Saving configuration 2023-12-12T00:46:32.992 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-12-12T00:46:32.997 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-12-12T00:46:33.023 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi134.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/yuriw-2023-12-11_23:27:14-rados-wip-yuri8-testing-2023-12-11-1101-distro-default-smithi/7487649', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-12-12 00:41:25.660326', 'locked_by': 'scheduled_yuriw@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNSwnXWJLVd/UmENlKG+RCgGnEAnwfxko3OHHXUu2ek5qu+LFa0GRhNbWUwK/TggSPq3XKdNfoiFGirPpVTk4is='} 2023-12-12T00:46:33.044 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi159.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/yuriw-2023-12-11_23:27:14-rados-wip-yuri8-testing-2023-12-11-1101-distro-default-smithi/7487649', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-12-12 00:41:25.658851', 'locked_by': 'scheduled_yuriw@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOToXyQy09qfcRgE2GT6B8E54p/tvsTo+NPPQXjPwHjrxPr0sjFm17JZ2Q8fFGzY/A6lwkqoAL9RIDx1r4qExRU='} 2023-12-12T00:46:33.044 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-12-12T00:46:33.054 INFO:teuthology.task.internal:roles: ubuntu@smithi134.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'mds.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2023-12-12T00:46:33.054 INFO:teuthology.task.internal:roles: ubuntu@smithi159.front.sepia.ceph.com - ['mon.b', 'mds.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2023-12-12T00:46:33.055 INFO:teuthology.run_tasks:Running task console_log... 2023-12-12T00:46:33.128 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f17c75a0e50>, signals=[15]) 2023-12-12T00:46:33.129 INFO:teuthology.run_tasks:Running task internal.connect... 2023-12-12T00:46:33.138 INFO:teuthology.task.internal:Opening connections... 2023-12-12T00:46:33.138 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi134.front.sepia.ceph.com 2023-12-12T00:46:33.141 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi134.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-12-12T00:46:33.209 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi159.front.sepia.ceph.com 2023-12-12T00:46:33.210 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi159.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-12-12T00:46:33.282 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-12-12T00:46:33.288 DEBUG:teuthology.orchestra.run.smithi134:> uname -m 2023-12-12T00:46:33.311 INFO:teuthology.orchestra.run.smithi134.stdout:x86_64 2023-12-12T00:46:33.311 DEBUG:teuthology.orchestra.run.smithi134:> cat /etc/os-release 2023-12-12T00:46:33.372 INFO:teuthology.orchestra.run.smithi134.stdout:NAME="CentOS Stream" 2023-12-12T00:46:33.372 INFO:teuthology.orchestra.run.smithi134.stdout:VERSION="8" 2023-12-12T00:46:33.372 INFO:teuthology.orchestra.run.smithi134.stdout:ID="centos" 2023-12-12T00:46:33.372 INFO:teuthology.orchestra.run.smithi134.stdout:ID_LIKE="rhel fedora" 2023-12-12T00:46:33.372 INFO:teuthology.orchestra.run.smithi134.stdout:VERSION_ID="8" 2023-12-12T00:46:33.373 INFO:teuthology.orchestra.run.smithi134.stdout:PLATFORM_ID="platform:el8" 2023-12-12T00:46:33.373 INFO:teuthology.orchestra.run.smithi134.stdout:PRETTY_NAME="CentOS Stream 8" 2023-12-12T00:46:33.373 INFO:teuthology.orchestra.run.smithi134.stdout:ANSI_COLOR="0;31" 2023-12-12T00:46:33.373 INFO:teuthology.orchestra.run.smithi134.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2023-12-12T00:46:33.373 INFO:teuthology.orchestra.run.smithi134.stdout:HOME_URL="https://centos.org/" 2023-12-12T00:46:33.373 INFO:teuthology.orchestra.run.smithi134.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-12-12T00:46:33.373 INFO:teuthology.orchestra.run.smithi134.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2023-12-12T00:46:33.373 INFO:teuthology.orchestra.run.smithi134.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2023-12-12T00:46:33.374 INFO:teuthology.lock.ops:Updating smithi134.front.sepia.ceph.com on lock server 2023-12-12T00:46:33.394 DEBUG:teuthology.orchestra.run.smithi159:> uname -m 2023-12-12T00:46:33.417 INFO:teuthology.orchestra.run.smithi159.stdout:x86_64 2023-12-12T00:46:33.417 DEBUG:teuthology.orchestra.run.smithi159:> cat /etc/os-release 2023-12-12T00:46:33.479 INFO:teuthology.orchestra.run.smithi159.stdout:NAME="CentOS Stream" 2023-12-12T00:46:33.479 INFO:teuthology.orchestra.run.smithi159.stdout:VERSION="8" 2023-12-12T00:46:33.479 INFO:teuthology.orchestra.run.smithi159.stdout:ID="centos" 2023-12-12T00:46:33.479 INFO:teuthology.orchestra.run.smithi159.stdout:ID_LIKE="rhel fedora" 2023-12-12T00:46:33.479 INFO:teuthology.orchestra.run.smithi159.stdout:VERSION_ID="8" 2023-12-12T00:46:33.479 INFO:teuthology.orchestra.run.smithi159.stdout:PLATFORM_ID="platform:el8" 2023-12-12T00:46:33.479 INFO:teuthology.orchestra.run.smithi159.stdout:PRETTY_NAME="CentOS Stream 8" 2023-12-12T00:46:33.480 INFO:teuthology.orchestra.run.smithi159.stdout:ANSI_COLOR="0;31" 2023-12-12T00:46:33.480 INFO:teuthology.orchestra.run.smithi159.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2023-12-12T00:46:33.480 INFO:teuthology.orchestra.run.smithi159.stdout:HOME_URL="https://centos.org/" 2023-12-12T00:46:33.480 INFO:teuthology.orchestra.run.smithi159.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-12-12T00:46:33.480 INFO:teuthology.orchestra.run.smithi159.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2023-12-12T00:46:33.480 INFO:teuthology.orchestra.run.smithi159.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2023-12-12T00:46:33.480 INFO:teuthology.lock.ops:Updating smithi159.front.sepia.ceph.com on lock server 2023-12-12T00:46:33.503 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-12-12T00:46:33.512 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-12-12T00:46:33.517 INFO:teuthology.task.internal:Checking for old test directory... 2023-12-12T00:46:33.517 DEBUG:teuthology.orchestra.run.smithi134:> test '!' -e /home/ubuntu/cephtest 2023-12-12T00:46:33.520 DEBUG:teuthology.orchestra.run.smithi159:> test '!' -e /home/ubuntu/cephtest 2023-12-12T00:46:33.540 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-12-12T00:46:33.547 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-12-12T00:46:33.547 DEBUG:teuthology.orchestra.run.smithi134:> test -z $(ls -A /var/lib/ceph) 2023-12-12T00:46:33.581 DEBUG:teuthology.orchestra.run.smithi159:> test -z $(ls -A /var/lib/ceph) 2023-12-12T00:46:33.608 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-12-12T00:46:33.664 INFO:teuthology.run_tasks:Running task kernel... 2023-12-12T00:46:33.678 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2023-12-12T00:46:33.678 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2023-12-12T00:46:33.679 DEBUG:teuthology.orchestra.run.smithi134:> test -f /run/.containerenv -o -f /.dockerenv 2023-12-12T00:46:33.679 DEBUG:teuthology.orchestra.run.smithi159:> test -f /run/.containerenv -o -f /.dockerenv 2023-12-12T00:46:33.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-12-12T00:46:33.699 DEBUG:teuthology.orchestra.run.smithi134:> uname -r 2023-12-12T00:46:33.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-12-12T00:46:33.700 DEBUG:teuthology.orchestra.run.smithi159:> uname -r 2023-12-12T00:46:33.758 INFO:teuthology.orchestra.run.smithi134.stdout:4.18.0-489.el8.x86_64 2023-12-12T00:46:33.759 INFO:teuthology.task.kernel:Running kernel on smithi134: 4.18.0-489.el8.x86_64 2023-12-12T00:46:33.759 DEBUG:teuthology.orchestra.run.smithi134:> sudo yum install -y kernel 2023-12-12T00:46:33.761 INFO:teuthology.orchestra.run.smithi159.stdout:4.18.0-489.el8.x86_64 2023-12-12T00:46:33.761 INFO:teuthology.task.kernel:Running kernel on smithi159: 4.18.0-489.el8.x86_64 2023-12-12T00:46:33.761 DEBUG:teuthology.orchestra.run.smithi159:> sudo yum install -y kernel 2023-12-12T00:46:37.237 INFO:teuthology.orchestra.run.smithi159.stdout:CentOS Stream 8 - AppStream 37 MB/s | 35 MB 00:00 2023-12-12T00:46:37.399 INFO:teuthology.orchestra.run.smithi134.stdout:CentOS Stream 8 - AppStream 32 MB/s | 35 MB 00:01 2023-12-12T00:46:44.596 INFO:teuthology.orchestra.run.smithi159.stdout:CentOS Stream 8 - BaseOS 51 MB/s | 56 MB 00:01 2023-12-12T00:46:45.049 INFO:teuthology.orchestra.run.smithi134.stdout:CentOS Stream 8 - BaseOS 44 MB/s | 56 MB 00:01 2023-12-12T00:46:52.688 INFO:teuthology.orchestra.run.smithi159.stdout:CentOS Stream 8 - Extras 131 kB/s | 18 kB 00:00 2023-12-12T00:46:53.287 INFO:teuthology.orchestra.run.smithi159.stdout:CentOS Stream 8 - Extras common packages 16 kB/s | 6.9 kB 00:00 2023-12-12T00:46:53.332 INFO:teuthology.orchestra.run.smithi134.stdout:CentOS Stream 8 - Extras 54 kB/s | 18 kB 00:00 2023-12-12T00:46:53.606 INFO:teuthology.orchestra.run.smithi159.stdout:Copr repo for python3-asyncssh owned by ceph 21 kB/s | 3.5 kB 00:00 2023-12-12T00:46:53.924 INFO:teuthology.orchestra.run.smithi134.stdout:CentOS Stream 8 - Extras common packages 17 kB/s | 6.9 kB 00:00 2023-12-12T00:46:54.209 INFO:teuthology.orchestra.run.smithi134.stdout:Copr repo for python3-asyncssh owned by ceph 26 kB/s | 3.5 kB 00:00 2023-12-12T00:46:54.967 INFO:teuthology.orchestra.run.smithi159.stdout:Extra Packages for Enterprise Linux 13 MB/s | 16 MB 00:01 2023-12-12T00:46:56.470 INFO:teuthology.orchestra.run.smithi134.stdout:Extra Packages for Enterprise Linux 7.6 MB/s | 16 MB 00:02 2023-12-12T00:46:58.840 INFO:teuthology.orchestra.run.smithi159.stdout:lab-extras 432 kB/s | 24 kB 00:00 2023-12-12T00:47:00.350 INFO:teuthology.orchestra.run.smithi134.stdout:lab-extras 288 kB/s | 24 kB 00:00 2023-12-12T00:47:02.498 INFO:teuthology.orchestra.run.smithi159.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2023-12-12T00:47:02.498 INFO:teuthology.orchestra.run.smithi159.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2023-12-12T00:47:02.817 INFO:teuthology.orchestra.run.smithi159.stdout:Dependencies resolved. 2023-12-12T00:47:02.818 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2023-12-12T00:47:02.818 INFO:teuthology.orchestra.run.smithi159.stdout: Package Architecture Version Repository Size 2023-12-12T00:47:02.818 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2023-12-12T00:47:02.818 INFO:teuthology.orchestra.run.smithi159.stdout:Installing: 2023-12-12T00:47:02.818 INFO:teuthology.orchestra.run.smithi159.stdout: kernel x86_64 4.18.0-527.el8 baseos 10 M 2023-12-12T00:47:02.818 INFO:teuthology.orchestra.run.smithi159.stdout:Installing dependencies: 2023-12-12T00:47:02.818 INFO:teuthology.orchestra.run.smithi159.stdout: kernel-core x86_64 4.18.0-527.el8 baseos 43 M 2023-12-12T00:47:02.819 INFO:teuthology.orchestra.run.smithi159.stdout: kernel-modules x86_64 4.18.0-527.el8 baseos 36 M 2023-12-12T00:47:02.819 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-12-12T00:47:02.819 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction Summary 2023-12-12T00:47:02.819 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2023-12-12T00:47:02.819 INFO:teuthology.orchestra.run.smithi159.stdout:Install 3 Packages 2023-12-12T00:47:02.819 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-12-12T00:47:02.819 INFO:teuthology.orchestra.run.smithi159.stdout:Total download size: 90 M 2023-12-12T00:47:02.820 INFO:teuthology.orchestra.run.smithi159.stdout:Installed size: 96 M 2023-12-12T00:47:02.820 INFO:teuthology.orchestra.run.smithi159.stdout:Downloading Packages: 2023-12-12T00:47:03.624 INFO:teuthology.orchestra.run.smithi159.stdout:(1/3): kernel-4.18.0-527.el8.x86_64.rpm 24 MB/s | 10 MB 00:00 2023-12-12T00:47:04.009 INFO:teuthology.orchestra.run.smithi134.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2023-12-12T00:47:04.009 INFO:teuthology.orchestra.run.smithi134.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2023-12-12T00:47:04.167 INFO:teuthology.orchestra.run.smithi159.stdout:(2/3): kernel-modules-4.18.0-527.el8.x86_64.rpm 38 MB/s | 36 MB 00:00 2023-12-12T00:47:04.291 INFO:teuthology.orchestra.run.smithi134.stdout:Dependencies resolved. 2023-12-12T00:47:04.292 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2023-12-12T00:47:04.292 INFO:teuthology.orchestra.run.smithi134.stdout: Package Architecture Version Repository Size 2023-12-12T00:47:04.292 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2023-12-12T00:47:04.292 INFO:teuthology.orchestra.run.smithi134.stdout:Installing: 2023-12-12T00:47:04.292 INFO:teuthology.orchestra.run.smithi134.stdout: kernel x86_64 4.18.0-527.el8 baseos 10 M 2023-12-12T00:47:04.292 INFO:teuthology.orchestra.run.smithi134.stdout:Installing dependencies: 2023-12-12T00:47:04.292 INFO:teuthology.orchestra.run.smithi134.stdout: kernel-core x86_64 4.18.0-527.el8 baseos 43 M 2023-12-12T00:47:04.292 INFO:teuthology.orchestra.run.smithi134.stdout: kernel-modules x86_64 4.18.0-527.el8 baseos 36 M 2023-12-12T00:47:04.292 INFO:teuthology.orchestra.run.smithi134.stdout: 2023-12-12T00:47:04.292 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction Summary 2023-12-12T00:47:04.292 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2023-12-12T00:47:04.293 INFO:teuthology.orchestra.run.smithi134.stdout:Install 3 Packages 2023-12-12T00:47:04.293 INFO:teuthology.orchestra.run.smithi134.stdout: 2023-12-12T00:47:04.293 INFO:teuthology.orchestra.run.smithi134.stdout:Total download size: 90 M 2023-12-12T00:47:04.293 INFO:teuthology.orchestra.run.smithi134.stdout:Installed size: 96 M 2023-12-12T00:47:04.293 INFO:teuthology.orchestra.run.smithi134.stdout:Downloading Packages: 2023-12-12T00:47:04.592 INFO:teuthology.orchestra.run.smithi159.stdout:(3/3): kernel-core-4.18.0-527.el8.x86_64.rpm 31 MB/s | 43 MB 00:01 2023-12-12T00:47:04.592 INFO:teuthology.orchestra.run.smithi159.stdout:-------------------------------------------------------------------------------- 2023-12-12T00:47:04.592 INFO:teuthology.orchestra.run.smithi159.stdout:Total 51 MB/s | 90 MB 00:01 2023-12-12T00:47:05.273 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction check 2023-12-12T00:47:05.335 INFO:teuthology.orchestra.run.smithi134.stdout:(1/3): kernel-4.18.0-527.el8.x86_64.rpm 24 MB/s | 10 MB 00:00 2023-12-12T00:47:05.690 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction check succeeded. 2023-12-12T00:47:05.690 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction test 2023-12-12T00:47:05.886 INFO:teuthology.orchestra.run.smithi134.stdout:(2/3): kernel-modules-4.18.0-527.el8.x86_64.rpm 37 MB/s | 36 MB 00:00 2023-12-12T00:47:06.311 INFO:teuthology.orchestra.run.smithi134.stdout:(3/3): kernel-core-4.18.0-527.el8.x86_64.rpm 31 MB/s | 43 MB 00:01 2023-12-12T00:47:06.311 INFO:teuthology.orchestra.run.smithi134.stdout:-------------------------------------------------------------------------------- 2023-12-12T00:47:06.311 INFO:teuthology.orchestra.run.smithi134.stdout:Total 44 MB/s | 90 MB 00:02 2023-12-12T00:47:06.937 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction test succeeded. 2023-12-12T00:47:06.959 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction 2023-12-12T00:47:06.991 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction check 2023-12-12T00:47:07.425 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction check succeeded. 2023-12-12T00:47:07.425 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction test 2023-12-12T00:47:08.656 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction test succeeded. 2023-12-12T00:47:08.684 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction 2023-12-12T00:47:09.761 INFO:teuthology.orchestra.run.smithi159.stdout: Preparing : 1/1 2023-12-12T00:47:10.262 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : kernel-core-4.18.0-527.el8.x86_64 1/3 2023-12-12T00:47:11.476 INFO:teuthology.orchestra.run.smithi134.stdout: Preparing : 1/1 2023-12-12T00:47:11.999 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : kernel-core-4.18.0-527.el8.x86_64 1/3 2023-12-12T00:47:13.813 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: kernel-core-4.18.0-527.el8.x86_64 1/3 2023-12-12T00:47:14.170 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : kernel-modules-4.18.0-527.el8.x86_64 2/3 2023-12-12T00:47:15.610 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: kernel-core-4.18.0-527.el8.x86_64 1/3 2023-12-12T00:47:16.486 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : kernel-modules-4.18.0-527.el8.x86_64 2/3 2023-12-12T00:47:17.595 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: kernel-modules-4.18.0-527.el8.x86_64 2/3 2023-12-12T00:47:17.885 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : kernel-4.18.0-527.el8.x86_64 3/3 2023-12-12T00:47:19.917 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: kernel-modules-4.18.0-527.el8.x86_64 2/3 2023-12-12T00:47:20.216 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : kernel-4.18.0-527.el8.x86_64 3/3 2023-12-12T00:47:39.725 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: kernel-core-4.18.0-527.el8.x86_64 3/3 2023-12-12T00:47:39.748 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: kernel-modules-4.18.0-527.el8.x86_64 3/3 2023-12-12T00:47:40.093 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: kernel-4.18.0-527.el8.x86_64 3/3 2023-12-12T00:47:40.093 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : kernel-4.18.0-527.el8.x86_64 1/3 2023-12-12T00:47:40.093 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : kernel-core-4.18.0-527.el8.x86_64 2/3 2023-12-12T00:47:40.856 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : kernel-modules-4.18.0-527.el8.x86_64 3/3 2023-12-12T00:47:40.856 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-12-12T00:47:40.856 INFO:teuthology.orchestra.run.smithi159.stdout:Installed: 2023-12-12T00:47:40.856 INFO:teuthology.orchestra.run.smithi159.stdout: kernel-4.18.0-527.el8.x86_64 kernel-core-4.18.0-527.el8.x86_64 2023-12-12T00:47:40.856 INFO:teuthology.orchestra.run.smithi159.stdout: kernel-modules-4.18.0-527.el8.x86_64 2023-12-12T00:47:40.856 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-12-12T00:47:40.856 INFO:teuthology.orchestra.run.smithi159.stdout:Complete! 2023-12-12T00:47:41.017 DEBUG:teuthology.orchestra.run.smithi159:> rpm -q kernel | sort -rV | head -n 1 2023-12-12T00:47:41.230 INFO:teuthology.orchestra.run.smithi159.stdout:kernel-4.18.0-527.el8.x86_64 2023-12-12T00:47:41.230 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-527.el8.x86_64 2023-12-12T00:47:41.230 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-527.el8.x86_64 2023-12-12T00:47:41.230 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2023-12-12T00:47:41.230 DEBUG:teuthology.task.kernel:src is distro, skipping download 2023-12-12T00:47:41.230 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2023-12-12T00:47:41.231 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi159.front.sepia.ceph.com, path=None, version=distro) 2023-12-12T00:47:41.231 DEBUG:teuthology.orchestra.run.smithi159:> rpm -q kernel | sort -rV | head -n 1 2023-12-12T00:47:41.439 INFO:teuthology.orchestra.run.smithi159.stdout:kernel-4.18.0-527.el8.x86_64 2023-12-12T00:47:41.440 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-527.el8.x86_64 2023-12-12T00:47:41.440 DEBUG:teuthology.orchestra.run.smithi159:> sudo rpm -qi grub2-tools 2023-12-12T00:47:41.473 INFO:teuthology.orchestra.run.smithi159.stdout:Name : grub2-tools 2023-12-12T00:47:41.474 INFO:teuthology.orchestra.run.smithi159.stdout:Epoch : 1 2023-12-12T00:47:41.474 INFO:teuthology.orchestra.run.smithi159.stdout:Version : 2.02 2023-12-12T00:47:41.474 INFO:teuthology.orchestra.run.smithi159.stdout:Release : 129.el8 2023-12-12T00:47:41.474 INFO:teuthology.orchestra.run.smithi159.stdout:Architecture: x86_64 2023-12-12T00:47:41.474 INFO:teuthology.orchestra.run.smithi159.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2023-12-12T00:47:41.474 INFO:teuthology.orchestra.run.smithi159.stdout:Group : System Environment/Base 2023-12-12T00:47:41.474 INFO:teuthology.orchestra.run.smithi159.stdout:Size : 9265586 2023-12-12T00:47:41.474 INFO:teuthology.orchestra.run.smithi159.stdout:License : GPLv3+ 2023-12-12T00:47:41.474 INFO:teuthology.orchestra.run.smithi159.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2023-12-12T00:47:41.474 INFO:teuthology.orchestra.run.smithi159.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2023-12-12T00:47:41.474 INFO:teuthology.orchestra.run.smithi159.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2023-12-12T00:47:41.474 INFO:teuthology.orchestra.run.smithi159.stdout:Build Host : kbuilder.bsys.centos.org 2023-12-12T00:47:41.474 INFO:teuthology.orchestra.run.smithi159.stdout:Relocations : (not relocatable) 2023-12-12T00:47:41.475 INFO:teuthology.orchestra.run.smithi159.stdout:Packager : CentOS BuildSystem 2023-12-12T00:47:41.475 INFO:teuthology.orchestra.run.smithi159.stdout:Vendor : CentOS 2023-12-12T00:47:41.475 INFO:teuthology.orchestra.run.smithi159.stdout:URL : http://www.gnu.org/software/grub/ 2023-12-12T00:47:41.475 INFO:teuthology.orchestra.run.smithi159.stdout:Summary : Support tools for GRUB. 2023-12-12T00:47:41.475 INFO:teuthology.orchestra.run.smithi159.stdout:Description : 2023-12-12T00:47:41.475 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-12-12T00:47:41.475 INFO:teuthology.orchestra.run.smithi159.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2023-12-12T00:47:41.475 INFO:teuthology.orchestra.run.smithi159.stdout:customizable bootloader with modular architecture. It supports a rich 2023-12-12T00:47:41.475 INFO:teuthology.orchestra.run.smithi159.stdout:variety of kernel formats, file systems, computer architectures and 2023-12-12T00:47:41.475 INFO:teuthology.orchestra.run.smithi159.stdout:hardware devices. 2023-12-12T00:47:41.475 INFO:teuthology.orchestra.run.smithi159.stdout: 2023-12-12T00:47:41.475 INFO:teuthology.orchestra.run.smithi159.stdout:This subpackage provides tools for support of all platforms. 2023-12-12T00:47:41.476 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2023-12-12T00:47:41.476 INFO:teuthology.task.kernel:Updating grub on smithi159 to boot 4.18.0-527.el8.x86_64 2023-12-12T00:47:41.476 DEBUG:teuthology.orchestra.run.smithi159:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2023-12-12T00:47:42.217 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: kernel-core-4.18.0-527.el8.x86_64 3/3 2023-12-12T00:47:42.242 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: kernel-modules-4.18.0-527.el8.x86_64 3/3 2023-12-12T00:47:42.587 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: kernel-4.18.0-527.el8.x86_64 3/3 2023-12-12T00:47:42.587 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : kernel-4.18.0-527.el8.x86_64 1/3 2023-12-12T00:47:42.587 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : kernel-core-4.18.0-527.el8.x86_64 2/3 2023-12-12T00:47:43.147 INFO:teuthology.orchestra.run.smithi159.stderr:Generating grub configuration file ... 2023-12-12T00:47:43.660 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : kernel-modules-4.18.0-527.el8.x86_64 3/3 2023-12-12T00:47:43.660 INFO:teuthology.orchestra.run.smithi134.stdout: 2023-12-12T00:47:43.660 INFO:teuthology.orchestra.run.smithi134.stdout:Installed: 2023-12-12T00:47:43.660 INFO:teuthology.orchestra.run.smithi134.stdout: kernel-4.18.0-527.el8.x86_64 kernel-core-4.18.0-527.el8.x86_64 2023-12-12T00:47:43.660 INFO:teuthology.orchestra.run.smithi134.stdout: kernel-modules-4.18.0-527.el8.x86_64 2023-12-12T00:47:43.661 INFO:teuthology.orchestra.run.smithi134.stdout: 2023-12-12T00:47:43.661 INFO:teuthology.orchestra.run.smithi134.stdout:Complete! 2023-12-12T00:47:43.806 DEBUG:teuthology.orchestra.run.smithi134:> rpm -q kernel | sort -rV | head -n 1 2023-12-12T00:47:44.023 INFO:teuthology.orchestra.run.smithi134.stdout:kernel-4.18.0-527.el8.x86_64 2023-12-12T00:47:44.023 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-527.el8.x86_64 2023-12-12T00:47:44.023 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-527.el8.x86_64 2023-12-12T00:47:44.024 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2023-12-12T00:47:44.024 DEBUG:teuthology.task.kernel:src is distro, skipping download 2023-12-12T00:47:44.024 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2023-12-12T00:47:44.024 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi134.front.sepia.ceph.com, path=None, version=distro) 2023-12-12T00:47:44.024 DEBUG:teuthology.orchestra.run.smithi134:> rpm -q kernel | sort -rV | head -n 1 2023-12-12T00:47:44.232 INFO:teuthology.orchestra.run.smithi134.stdout:kernel-4.18.0-527.el8.x86_64 2023-12-12T00:47:44.233 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-527.el8.x86_64 2023-12-12T00:47:44.233 DEBUG:teuthology.orchestra.run.smithi134:> sudo rpm -qi grub2-tools 2023-12-12T00:47:44.267 INFO:teuthology.orchestra.run.smithi134.stdout:Name : grub2-tools 2023-12-12T00:47:44.267 INFO:teuthology.orchestra.run.smithi134.stdout:Epoch : 1 2023-12-12T00:47:44.267 INFO:teuthology.orchestra.run.smithi134.stdout:Version : 2.02 2023-12-12T00:47:44.267 INFO:teuthology.orchestra.run.smithi134.stdout:Release : 129.el8 2023-12-12T00:47:44.268 INFO:teuthology.orchestra.run.smithi134.stdout:Architecture: x86_64 2023-12-12T00:47:44.268 INFO:teuthology.orchestra.run.smithi134.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2023-12-12T00:47:44.268 INFO:teuthology.orchestra.run.smithi134.stdout:Group : System Environment/Base 2023-12-12T00:47:44.268 INFO:teuthology.orchestra.run.smithi134.stdout:Size : 9265586 2023-12-12T00:47:44.268 INFO:teuthology.orchestra.run.smithi134.stdout:License : GPLv3+ 2023-12-12T00:47:44.268 INFO:teuthology.orchestra.run.smithi134.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2023-12-12T00:47:44.268 INFO:teuthology.orchestra.run.smithi134.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2023-12-12T00:47:44.268 INFO:teuthology.orchestra.run.smithi134.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2023-12-12T00:47:44.268 INFO:teuthology.orchestra.run.smithi134.stdout:Build Host : kbuilder.bsys.centos.org 2023-12-12T00:47:44.268 INFO:teuthology.orchestra.run.smithi134.stdout:Relocations : (not relocatable) 2023-12-12T00:47:44.268 INFO:teuthology.orchestra.run.smithi134.stdout:Packager : CentOS BuildSystem 2023-12-12T00:47:44.268 INFO:teuthology.orchestra.run.smithi134.stdout:Vendor : CentOS 2023-12-12T00:47:44.268 INFO:teuthology.orchestra.run.smithi134.stdout:URL : http://www.gnu.org/software/grub/ 2023-12-12T00:47:44.268 INFO:teuthology.orchestra.run.smithi134.stdout:Summary : Support tools for GRUB. 2023-12-12T00:47:44.269 INFO:teuthology.orchestra.run.smithi134.stdout:Description : 2023-12-12T00:47:44.269 INFO:teuthology.orchestra.run.smithi134.stdout: 2023-12-12T00:47:44.269 INFO:teuthology.orchestra.run.smithi134.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2023-12-12T00:47:44.269 INFO:teuthology.orchestra.run.smithi134.stdout:customizable bootloader with modular architecture. It supports a rich 2023-12-12T00:47:44.269 INFO:teuthology.orchestra.run.smithi134.stdout:variety of kernel formats, file systems, computer architectures and 2023-12-12T00:47:44.269 INFO:teuthology.orchestra.run.smithi134.stdout:hardware devices. 2023-12-12T00:47:44.269 INFO:teuthology.orchestra.run.smithi134.stdout: 2023-12-12T00:47:44.269 INFO:teuthology.orchestra.run.smithi134.stdout:This subpackage provides tools for support of all platforms. 2023-12-12T00:47:44.270 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2023-12-12T00:47:44.271 INFO:teuthology.task.kernel:Updating grub on smithi134 to boot 4.18.0-527.el8.x86_64 2023-12-12T00:47:44.271 DEBUG:teuthology.orchestra.run.smithi134:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2023-12-12T00:47:45.375 INFO:teuthology.orchestra.run.smithi159.stderr:done 2023-12-12T00:47:45.377 DEBUG:teuthology.orchestra.run.smithi159:> mktemp 2023-12-12T00:47:45.409 INFO:teuthology.orchestra.run.smithi159.stdout:/tmp/tmp.SFvsbUv6ZA 2023-12-12T00:47:45.410 DEBUG:teuthology.orchestra.run.smithi159:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.SFvsbUv6ZA 2023-12-12T00:47:45.484 DEBUG:teuthology.orchestra.run.smithi159:> sudo chmod 0666 /tmp/tmp.SFvsbUv6ZA 2023-12-12T00:47:45.625 INFO:teuthology.orchestra.run.smithi134.stderr:Generating grub configuration file ... 2023-12-12T00:47:45.629 DEBUG:teuthology.orchestra.remote:smithi159:/tmp/tmp.SFvsbUv6ZA is 6KB 2023-12-12T00:47:45.677 DEBUG:teuthology.orchestra.run.smithi159:> rm -fr /tmp/tmp.SFvsbUv6ZA 2023-12-12T00:47:45.694 DEBUG:teuthology.orchestra.run.smithi159:> sudo /bin/ls /boot/loader/entries || true 2023-12-12T00:47:45.770 INFO:teuthology.orchestra.run.smithi159.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2023-12-12T00:47:45.771 INFO:teuthology.orchestra.run.smithi159.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2023-12-12T00:47:45.771 INFO:teuthology.orchestra.run.smithi159.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2023-12-12T00:47:45.771 INFO:teuthology.orchestra.run.smithi159.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-527.el8.x86_64.conf 2023-12-12T00:47:45.772 DEBUG:teuthology.orchestra.run.smithi159:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-527.el8.x86_64 2023-12-12T00:47:45.920 DEBUG:teuthology.orchestra.run.smithi159:> sudo shutdown -r now 2023-12-12T00:47:48.108 INFO:teuthology.orchestra.run.smithi134.stderr:done 2023-12-12T00:47:48.109 DEBUG:teuthology.orchestra.run.smithi134:> mktemp 2023-12-12T00:47:48.142 INFO:teuthology.orchestra.run.smithi134.stdout:/tmp/tmp.4gGogxd94x 2023-12-12T00:47:48.142 DEBUG:teuthology.orchestra.run.smithi134:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.4gGogxd94x 2023-12-12T00:47:48.216 DEBUG:teuthology.orchestra.run.smithi134:> sudo chmod 0666 /tmp/tmp.4gGogxd94x 2023-12-12T00:47:48.361 DEBUG:teuthology.orchestra.remote:smithi134:/tmp/tmp.4gGogxd94x is 6KB 2023-12-12T00:47:48.410 DEBUG:teuthology.orchestra.run.smithi134:> rm -fr /tmp/tmp.4gGogxd94x 2023-12-12T00:47:48.427 DEBUG:teuthology.orchestra.run.smithi134:> sudo /bin/ls /boot/loader/entries || true 2023-12-12T00:47:48.507 INFO:teuthology.orchestra.run.smithi134.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2023-12-12T00:47:48.508 INFO:teuthology.orchestra.run.smithi134.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2023-12-12T00:47:48.508 INFO:teuthology.orchestra.run.smithi134.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2023-12-12T00:47:48.508 INFO:teuthology.orchestra.run.smithi134.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-527.el8.x86_64.conf 2023-12-12T00:47:48.509 DEBUG:teuthology.orchestra.run.smithi134:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-527.el8.x86_64 2023-12-12T00:47:48.692 DEBUG:teuthology.orchestra.run.smithi134:> sudo shutdown -r now 2023-12-12T00:48:15.950 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2023-12-12T00:48:15.951 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi159.front.sepia.ceph.com' 2023-12-12T00:48:15.952 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi159.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-12-12T00:48:18.697 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2023-12-12T00:48:18.698 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi134.front.sepia.ceph.com' 2023-12-12T00:48:18.698 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi134.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-12-12T00:48:34.406 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.159 2023-12-12T00:48:37.222 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.134 2023-12-12T00:48:43.410 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi159.front.sepia.ceph.com' 2023-12-12T00:48:43.411 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi159.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-12-12T00:48:46.226 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi134.front.sepia.ceph.com' 2023-12-12T00:48:46.226 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi134.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-12-12T00:48:46.470 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.159 2023-12-12T00:48:49.282 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.134 2023-12-12T00:48:58.478 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi159.front.sepia.ceph.com' 2023-12-12T00:48:58.479 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi159.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-12-12T00:49:01.286 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi134.front.sepia.ceph.com' 2023-12-12T00:49:01.287 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi134.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-12-12T00:49:16.902 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.159 2023-12-12T00:49:31.914 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi159.front.sepia.ceph.com' 2023-12-12T00:49:31.915 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi159.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-12-12T00:49:32.216 DEBUG:teuthology.orchestra.run.smithi159:> true 2023-12-12T00:49:32.552 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi159.front.sepia.ceph.com' 2023-12-12T00:49:32.552 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "4.18.0-527.el8.x86_64"... 2023-12-12T00:49:32.552 DEBUG:teuthology.orchestra.run.smithi159:> uname -r 2023-12-12T00:49:32.569 INFO:teuthology.orchestra.run.smithi159.stdout:4.18.0-527.el8.x86_64 2023-12-12T00:49:32.570 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-527.el8.x86_64 vs 4.18.0-527.el8.x86_64 2023-12-12T00:49:32.570 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-12-12T00:49:32.570 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2023-12-12T00:49:33.506 DEBUG:teuthology.orchestra.run.smithi134:> true 2023-12-12T00:49:33.571 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2023-12-12T00:49:33.572 DEBUG:teuthology.orchestra.run.smithi159:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-12-12T00:49:33.654 INFO:teuthology.orchestra.run.smithi159.stdout:ttyS1 2023-12-12T00:49:33.667 DEBUG:teuthology.parallel:result is None 2023-12-12T00:49:34.690 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi134.front.sepia.ceph.com' 2023-12-12T00:49:34.690 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "4.18.0-527.el8.x86_64"... 2023-12-12T00:49:34.691 DEBUG:teuthology.orchestra.run.smithi134:> uname -r 2023-12-12T00:49:35.074 INFO:teuthology.orchestra.run.smithi134.stdout:4.18.0-527.el8.x86_64 2023-12-12T00:49:35.074 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-527.el8.x86_64 vs 4.18.0-527.el8.x86_64 2023-12-12T00:49:35.074 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-12-12T00:49:35.074 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2023-12-12T00:49:36.075 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2023-12-12T00:49:36.075 DEBUG:teuthology.orchestra.run.smithi134:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-12-12T00:49:36.122 INFO:teuthology.orchestra.run.smithi134.stdout:ttyS1 2023-12-12T00:49:36.143 DEBUG:teuthology.parallel:result is None 2023-12-12T00:49:36.143 INFO:teuthology.run_tasks:Running task internal.base... 2023-12-12T00:49:36.158 INFO:teuthology.task.internal:Creating test directory... 2023-12-12T00:49:36.158 DEBUG:teuthology.orchestra.run.smithi134:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-12-12T00:49:36.160 DEBUG:teuthology.orchestra.run.smithi159:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-12-12T00:49:36.178 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-12-12T00:49:36.186 INFO:teuthology.run_tasks:Running task internal.archive... 2023-12-12T00:49:36.193 INFO:teuthology.task.internal:Creating archive directory... 2023-12-12T00:49:36.193 DEBUG:teuthology.orchestra.run.smithi134:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-12-12T00:49:36.217 DEBUG:teuthology.orchestra.run.smithi159:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-12-12T00:49:36.245 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-12-12T00:49:36.252 INFO:teuthology.task.internal:Enabling coredump saving... 2023-12-12T00:49:36.252 DEBUG:teuthology.orchestra.run.smithi134:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-12-12T00:49:36.283 DEBUG:teuthology.orchestra.run.smithi159:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-12-12T00:49:36.310 INFO:teuthology.orchestra.run.smithi134.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-12-12T00:49:36.319 INFO:teuthology.orchestra.run.smithi134.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-12-12T00:49:36.326 INFO:teuthology.orchestra.run.smithi159.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-12-12T00:49:36.336 INFO:teuthology.orchestra.run.smithi159.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-12-12T00:49:36.337 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-12-12T00:49:36.346 INFO:teuthology.task.internal:Configuring sudo... 2023-12-12T00:49:36.346 DEBUG:teuthology.orchestra.run.smithi134:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-12-12T00:49:36.364 DEBUG:teuthology.orchestra.run.smithi159:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-12-12T00:49:36.390 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-12-12T00:49:36.403 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-12-12T00:49:36.403 DEBUG:teuthology.orchestra.run.smithi134:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-12-12T00:49:36.430 DEBUG:teuthology.orchestra.run.smithi159:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-12-12T00:49:36.447 DEBUG:teuthology.orchestra.run.smithi134:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-12-12T00:49:36.533 DEBUG:teuthology.orchestra.run.smithi134:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-12-12T00:49:36.581 DEBUG:teuthology.orchestra.run.smithi134:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-12-12T00:49:36.666 DEBUG:teuthology.orchestra.run.smithi134:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-12-12T00:49:36.709 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2023-12-12T00:49:36.709 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-12-12T00:49:36.774 DEBUG:teuthology.orchestra.run.smithi159:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-12-12T00:49:36.816 DEBUG:teuthology.orchestra.run.smithi159:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-12-12T00:49:36.867 DEBUG:teuthology.orchestra.run.smithi159:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-12-12T00:49:36.951 DEBUG:teuthology.orchestra.run.smithi159:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-12-12T00:49:36.990 DEBUG:teuthology.orchestra.run.smithi159:> set -ex 2023-12-12T00:49:36.990 DEBUG:teuthology.orchestra.run.smithi159:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-12-12T00:49:37.054 DEBUG:teuthology.orchestra.run.smithi134:> sudo service rsyslog restart 2023-12-12T00:49:37.057 DEBUG:teuthology.orchestra.run.smithi159:> sudo service rsyslog restart 2023-12-12T00:49:37.109 INFO:teuthology.orchestra.run.smithi134.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-12-12T00:49:37.171 INFO:teuthology.orchestra.run.smithi159.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-12-12T00:49:37.485 INFO:teuthology.run_tasks:Running task internal.timer... 2023-12-12T00:49:37.494 INFO:teuthology.task.internal:Starting timer... 2023-12-12T00:49:37.494 INFO:teuthology.run_tasks:Running task pcp... 2023-12-12T00:49:37.508 INFO:teuthology.run_tasks:Running task selinux... 2023-12-12T00:49:37.523 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2023-12-12T00:49:37.524 DEBUG:teuthology.orchestra.run.smithi134:> sudo service auditd rotate 2023-12-12T00:49:37.622 INFO:teuthology.orchestra.run.smithi134.stdout:Rotating logs: 2023-12-12T00:49:37.624 DEBUG:teuthology.orchestra.run.smithi159:> sudo service auditd rotate 2023-12-12T00:49:37.781 INFO:teuthology.orchestra.run.smithi159.stdout:Rotating logs: 2023-12-12T00:49:37.788 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-12-12T00:49:37.789 DEBUG:teuthology.orchestra.run.smithi134:> /usr/sbin/getenforce 2023-12-12T00:49:37.817 INFO:teuthology.orchestra.run.smithi134.stdout:Permissive 2023-12-12T00:49:37.817 DEBUG:teuthology.orchestra.run.smithi159:> /usr/sbin/getenforce 2023-12-12T00:49:37.876 INFO:teuthology.orchestra.run.smithi159.stdout:Permissive 2023-12-12T00:49:37.876 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi134.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi159.front.sepia.ceph.com': 'permissive'} 2023-12-12T00:49:37.876 DEBUG:teuthology.orchestra.run.smithi134:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-12-12T00:49:37.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-12-12T00:49:37.904 DEBUG:teuthology.orchestra.run.smithi159:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-12-12T00:49:37.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-12-12T00:49:37.946 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-12-12T00:49:37.950 DEBUG:teuthology.orchestra.run.smithi134:> sudo /usr/sbin/setenforce permissive 2023-12-12T00:49:37.976 DEBUG:teuthology.orchestra.run.smithi159:> sudo /usr/sbin/setenforce permissive 2023-12-12T00:49:38.270 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-12-12T00:49:38.280 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2023-12-12T00:49:38.287 INFO:teuthology.repo_utils:Fetching main from origin 2023-12-12T00:49:38.414 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-12-12T00:49:38.433 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-12-12T00:49:38.434 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi134.front.sepia.ceph.com,smithi159.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-12-12T00:55:25.485 INFO:teuthology.task.ansible:Archiving ansible failure log at: /home/teuthworker/archive/yuriw-2023-12-11_23:27:14-rados-wip-yuri8-testing-2023-12-11-1101-distro-default-smithi/7487649/ansible_failures.yaml 2023-12-12T00:55:25.488 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_5f3731ee8689ef3b0ef7d571e4cc2cb968c78d73/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/home/teuthworker/src/git.ceph.com_teuthology_5f3731ee8689ef3b0ef7d571e4cc2cb968c78d73/teuthology/task/__init__.py", line 123, in __enter__ self.begin() File "/home/teuthworker/src/git.ceph.com_teuthology_5f3731ee8689ef3b0ef7d571e4cc2cb968c78d73/teuthology/task/ansible.py", line 503, in begin super(CephLab, self).begin() File "/home/teuthworker/src/git.ceph.com_teuthology_5f3731ee8689ef3b0ef7d571e4cc2cb968c78d73/teuthology/task/ansible.py", line 341, in begin self.execute_playbook() File "/home/teuthworker/src/git.ceph.com_teuthology_5f3731ee8689ef3b0ef7d571e4cc2cb968c78d73/teuthology/task/ansible.py", line 369, in execute_playbook self._handle_failure(command, status) File "/home/teuthworker/src/git.ceph.com_teuthology_5f3731ee8689ef3b0ef7d571e4cc2cb968c78d73/teuthology/task/ansible.py", line 397, in _handle_failure raise AnsibleFailedError(failures) teuthology.exceptions.AnsibleFailedError: [',', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Request to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64 failed: 500 - HTTP Error 500: INTERNAL SERVER ERROR', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at'] 2023-12-12T00:55:25.578 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=773c603e6db74db69a3fb21e32d966f4 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_5f3731ee8689ef3b0ef7d571e4cc2cb968c78d73/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/home/teuthworker/src/git.ceph.com_teuthology_5f3731ee8689ef3b0ef7d571e4cc2cb968c78d73/teuthology/task/__init__.py", line 123, in __enter__ self.begin() File "/home/teuthworker/src/git.ceph.com_teuthology_5f3731ee8689ef3b0ef7d571e4cc2cb968c78d73/teuthology/task/ansible.py", line 503, in begin super(CephLab, self).begin() File "/home/teuthworker/src/git.ceph.com_teuthology_5f3731ee8689ef3b0ef7d571e4cc2cb968c78d73/teuthology/task/ansible.py", line 341, in begin self.execute_playbook() File "/home/teuthworker/src/git.ceph.com_teuthology_5f3731ee8689ef3b0ef7d571e4cc2cb968c78d73/teuthology/task/ansible.py", line 369, in execute_playbook self._handle_failure(command, status) File "/home/teuthworker/src/git.ceph.com_teuthology_5f3731ee8689ef3b0ef7d571e4cc2cb968c78d73/teuthology/task/ansible.py", line 397, in _handle_failure raise AnsibleFailedError(failures) teuthology.exceptions.AnsibleFailedError: [',', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Request to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64 failed: 500 - HTTP Error 500: INTERNAL SERVER ERROR', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at'] 2023-12-12T00:55:25.580 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2023-12-12T00:55:25.588 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2023-12-12T00:55:25.588 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2023-12-12T00:55:25.662 DEBUG:teuthology.orchestra.run.smithi134:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-12-12T00:55:25.765 DEBUG:teuthology.orchestra.run.smithi159:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-12-12T00:55:25.852 DEBUG:teuthology.orchestra.run.smithi134:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-12-12T00:55:25.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-12-12T00:55:25.885 DEBUG:teuthology.orchestra.run.smithi159:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-12-12T00:55:25.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-12-12T00:55:25.918 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2023-12-12T00:55:25.926 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2023-12-12T00:55:25.935 INFO:teuthology.task.internal:Duration was 348.440706 seconds 2023-12-12T00:55:25.936 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2023-12-12T00:55:25.960 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2023-12-12T00:55:25.961 DEBUG:teuthology.orchestra.run.smithi134:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-12-12T00:55:25.963 DEBUG:teuthology.orchestra.run.smithi159:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-12-12T00:55:26.009 INFO:teuthology.orchestra.run.smithi134.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-12-12T00:55:26.010 INFO:teuthology.orchestra.run.smithi159.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-12-12T00:55:26.486 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2023-12-12T00:55:26.486 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi134.front.sepia.ceph.com 2023-12-12T00:55:26.486 DEBUG:teuthology.orchestra.run.smithi134:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-12-12T00:55:26.511 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi159.front.sepia.ceph.com 2023-12-12T00:55:26.511 DEBUG:teuthology.orchestra.run.smithi159:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-12-12T00:55:26.535 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2023-12-12T00:55:26.535 DEBUG:teuthology.orchestra.run.smithi134:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-12-12T00:55:26.553 DEBUG:teuthology.orchestra.run.smithi159:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-12-12T00:55:26.611 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2023-12-12T00:55:26.612 DEBUG:teuthology.orchestra.run.smithi134:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-12-12T00:55:26.632 DEBUG:teuthology.orchestra.run.smithi159:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-12-12T00:55:26.711 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2023-12-12T00:55:26.721 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2023-12-12T00:55:26.721 DEBUG:teuthology.orchestra.run.smithi134:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-12-12T00:55:26.749 DEBUG:teuthology.orchestra.run.smithi159:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-12-12T00:55:26.778 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2023-12-12T00:55:26.788 DEBUG:teuthology.orchestra.run.smithi134:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-12-12T00:55:26.791 DEBUG:teuthology.orchestra.run.smithi159:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-12-12T00:55:26.815 INFO:teuthology.orchestra.run.smithi134.stdout:kernel.core_pattern = core 2023-12-12T00:55:26.844 INFO:teuthology.orchestra.run.smithi159.stdout:kernel.core_pattern = core 2023-12-12T00:55:26.883 DEBUG:teuthology.orchestra.run.smithi134:> test -e /home/ubuntu/cephtest/archive/coredump 2023-12-12T00:55:26.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-12-12T00:55:26.901 DEBUG:teuthology.orchestra.run.smithi159:> test -e /home/ubuntu/cephtest/archive/coredump 2023-12-12T00:55:26.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-12-12T00:55:26.941 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2023-12-12T00:55:26.949 INFO:teuthology.task.internal:Transferring archived files... 2023-12-12T00:55:26.951 DEBUG:teuthology.misc:Transferring archived files from smithi134:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/yuriw-2023-12-11_23:27:14-rados-wip-yuri8-testing-2023-12-11-1101-distro-default-smithi/7487649/remote/smithi134 2023-12-12T00:55:26.952 DEBUG:teuthology.orchestra.run.smithi134:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-12-12T00:55:27.019 DEBUG:teuthology.misc:Transferring archived files from smithi159:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/yuriw-2023-12-11_23:27:14-rados-wip-yuri8-testing-2023-12-11-1101-distro-default-smithi/7487649/remote/smithi159 2023-12-12T00:55:27.020 DEBUG:teuthology.orchestra.run.smithi159:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-12-12T00:55:27.071 INFO:teuthology.task.internal:Removing archive directory... 2023-12-12T00:55:27.071 DEBUG:teuthology.orchestra.run.smithi134:> rm -rf -- /home/ubuntu/cephtest/archive 2023-12-12T00:55:27.074 DEBUG:teuthology.orchestra.run.smithi159:> rm -rf -- /home/ubuntu/cephtest/archive 2023-12-12T00:55:27.122 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2023-12-12T00:55:27.133 INFO:teuthology.task.internal:Not uploading archives. 2023-12-12T00:55:27.133 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2023-12-12T00:55:27.141 INFO:teuthology.task.internal:Tidying up after the test... 2023-12-12T00:55:27.141 DEBUG:teuthology.orchestra.run.smithi134:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-12-12T00:55:27.145 DEBUG:teuthology.orchestra.run.smithi159:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-12-12T00:55:27.161 INFO:teuthology.orchestra.run.smithi134.stdout: 525359 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Dec 12 00:55 /home/ubuntu/cephtest 2023-12-12T00:55:27.179 INFO:teuthology.orchestra.run.smithi159.stdout: 525838 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Dec 12 00:55 /home/ubuntu/cephtest 2023-12-12T00:55:27.181 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2023-12-12T00:55:27.191 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2023-12-12T00:55:27.210 INFO:teuthology.nuke:Checking targets against current locks 2023-12-12T00:55:27.241 DEBUG:teuthology.nuke:shortname: smithi134 2023-12-12T00:55:27.241 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-12-12T00:55:27.266 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi134.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/yuriw-2023-12-11_23:27:14-rados-wip-yuri8-testing-2023-12-11-1101-distro-default-smithi/7487649', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-12-12 00:41:25.660326', 'locked_by': 'scheduled_yuriw@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNSwnXWJLVd/UmENlKG+RCgGnEAnwfxko3OHHXUu2ek5qu+LFa0GRhNbWUwK/TggSPq3XKdNfoiFGirPpVTk4is='} 2023-12-12T00:55:27.269 DEBUG:teuthology.nuke:shortname: smithi159 2023-12-12T00:55:27.269 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-12-12T00:55:27.295 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi159.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/yuriw-2023-12-11_23:27:14-rados-wip-yuri8-testing-2023-12-11-1101-distro-default-smithi/7487649', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-12-12 00:41:25.658851', 'locked_by': 'scheduled_yuriw@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOToXyQy09qfcRgE2GT6B8E54p/tvsTo+NPPQXjPwHjrxPr0sjFm17JZ2Q8fFGzY/A6lwkqoAL9RIDx1r4qExRU='} 2023-12-12T00:55:27.326 INFO:teuthology.orchestra.console.smithi134:Power off 2023-12-12T00:55:27.327 DEBUG:teuthology.orchestra.console.smithi134:pexpect command: ipmitool -H smithi134.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-12-12T00:55:27.355 DEBUG:teuthology.orchestra.console.smithi134:power off output: Chassis Power Control: Down/Off 2023-12-12T00:55:27.356 DEBUG:teuthology.orchestra.console.smithi134:pexpect command: ipmitool -H smithi134.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-12-12T00:55:27.373 INFO:teuthology.orchestra.console.smithi159:Power off 2023-12-12T00:55:27.374 DEBUG:teuthology.orchestra.console.smithi159:pexpect command: ipmitool -H smithi159.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-12-12T00:55:27.389 DEBUG:teuthology.orchestra.console.smithi134:check power output: Chassis Power is on 2023-12-12T00:55:27.406 DEBUG:teuthology.orchestra.console.smithi159:power off output: Chassis Power Control: Down/Off 2023-12-12T00:55:27.406 DEBUG:teuthology.orchestra.console.smithi159:pexpect command: ipmitool -H smithi159.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-12-12T00:55:27.430 DEBUG:teuthology.orchestra.console.smithi159:check power output: Chassis Power is on 2023-12-12T00:55:31.391 DEBUG:teuthology.orchestra.console.smithi134:pexpect command: ipmitool -H smithi134.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-12-12T00:55:31.431 DEBUG:teuthology.orchestra.console.smithi159:pexpect command: ipmitool -H smithi159.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-12-12T00:55:31.504 DEBUG:teuthology.orchestra.console.smithi134:check power output: Chassis Power is on 2023-12-12T00:55:31.542 DEBUG:teuthology.orchestra.console.smithi159:check power output: Chassis Power is on 2023-12-12T00:55:35.509 DEBUG:teuthology.orchestra.console.smithi134:pexpect command: ipmitool -H smithi134.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-12-12T00:55:35.543 DEBUG:teuthology.orchestra.console.smithi159:pexpect command: ipmitool -H smithi159.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-12-12T00:55:35.620 DEBUG:teuthology.orchestra.console.smithi134:check power output: Chassis Power is on 2023-12-12T00:55:35.653 DEBUG:teuthology.orchestra.console.smithi159:check power output: Chassis Power is off 2023-12-12T00:55:35.754 INFO:teuthology.orchestra.console.smithi159:Power off completed 2023-12-12T00:55:39.624 DEBUG:teuthology.orchestra.console.smithi134:pexpect command: ipmitool -H smithi134.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-12-12T00:55:39.735 DEBUG:teuthology.orchestra.console.smithi134:check power output: Chassis Power is off 2023-12-12T00:55:39.835 INFO:teuthology.orchestra.console.smithi134:Power off completed 2023-12-12T00:55:39.941 INFO:teuthology.run:Summary data: description: rados/upgrade/parallel/{0-random-distro$/{centos_8.stream_container_tools} 0-start 1-tasks mon_election/classic upgrade-sequence workload/{ec-rados-default rados_api rados_loadgenbig rbd_import_export test_rbd_api test_rbd_python}} duration: 348.440705537796 failure_reason: '['','', ''Bugzilla. In case of problems, contact the owner of this repository.'', ''Enabling a Copr repository. Please note that this repository is not part'', ''Error: Request to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64 failed: 500 - HTTP Error 500: INTERNAL SERVER ERROR'', ''Please do not file bug reports about these packages in Fedora'', ''The Fedora Project does not exercise any power over the contents of'', ''and packages are not held to any quality or security level.'', ''of the main distribution, and quality may vary.'', ''this repository beyond the rules outlined in the Copr FAQ at'']' owner: scheduled_yuriw@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=773c603e6db74db69a3fb21e32d966f4 status: dead success: false 2023-12-12T00:55:39.942 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-12-12T00:55:40.030 INFO:teuthology.run:DEAD