2024-04-10T22:38:15.382 INFO:root:teuthology version: 0.0.1.dev267+g6c63784 2024-04-10T22:38:15.383 DEBUG:teuthology.run:Teuthology command: teuthology --archive /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650176 --verbose --name teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi --owner scheduled_teuthology@teuthology --description upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_rhel8 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} -- /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650176/orig.config.yaml 2024-04-10T22:38:15.438 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-10T22:38:15.535 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650176 branch: reef description: upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_rhel8 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} email: ceph-qa@ceph.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' first_in_suite: false job_id: '7650176' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run basic import/export cli tests for rbd ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: rhel os_version: '8.6' overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: reef sha1: 4f70a9a90881107996c8b7a3e89ba8b32624713c owner: scheduled_teuthology@teuthology priority: 830 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 4604 sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: pacific clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 0/1 suite: upgrade:pacific-x suite_branch: reef suite_path: /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 4f70a9a90881107996c8b7a3e89ba8b32624713c targets: smithi139.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNdCv3N7o0KROGjUFTqcHTfLRaICJWZtjVlBi4CcJyOx2HmXG3vfT/4YBtgPZvNdPxnRX5uZxD3MfQwA8ney8FQ= smithi172.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBP9QiZiA/cNy65vCOuDm4Qc0T042szbUGjKOAPrzFpdTD0xwkuJp4Nb5IIk6yFV+VkmLtTMRNlu1gagTgtie8n0= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools:rhel8 --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - install: branch: pacific exclude_packages: - ceph-volume - cephadm: cephadm_branch: pacific cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:pacific - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 6c637841c215537a4502385240412f1966e0faab timestamp: 2024-04-10_01:08:08 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-04-10T22:38:15.535 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa; will attempt to use it 2024-04-10T22:38:15.535 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa/tasks 2024-04-10T22:38:15.536 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-10T22:38:15.537 INFO:teuthology.task.internal:Checking packages... 2024-04-10T22:38:15.558 INFO:teuthology.task.internal:Checking packages for os_type 'rhel', flavor 'default' and ceph hash 'd540ebaca6b131a1dd560e7f69e024b133bbaa42' 2024-04-10T22:38:15.558 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-10T22:38:15.558 INFO:teuthology.packaging:ref: None 2024-04-10T22:38:15.558 INFO:teuthology.packaging:tag: None 2024-04-10T22:38:15.558 INFO:teuthology.packaging:branch: reef 2024-04-10T22:38:15.558 INFO:teuthology.packaging:sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 2024-04-10T22:38:15.558 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef 2024-04-10T22:38:15.923 INFO:teuthology.task.internal:Found packages for ceph version 18.2.2-1189.gc96c8d47 2024-04-10T22:38:15.925 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-10T22:38:15.934 INFO:teuthology.task.internal:no buildpackages task found 2024-04-10T22:38:15.934 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-10T22:38:15.940 INFO:teuthology.task.internal:Saving configuration 2024-04-10T22:38:15.961 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-10T22:38:15.966 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T22:38:15.991 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi139.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650176', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:13.867513', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNdCv3N7o0KROGjUFTqcHTfLRaICJWZtjVlBi4CcJyOx2HmXG3vfT/4YBtgPZvNdPxnRX5uZxD3MfQwA8ney8FQ='} 2024-04-10T22:38:16.013 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi172.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650176', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:13.868841', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBP9QiZiA/cNy65vCOuDm4Qc0T042szbUGjKOAPrzFpdTD0xwkuJp4Nb5IIk6yFV+VkmLtTMRNlu1gagTgtie8n0='} 2024-04-10T22:38:16.013 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-10T22:38:16.019 INFO:teuthology.task.internal:roles: ubuntu@smithi139.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-04-10T22:38:16.019 INFO:teuthology.task.internal:roles: ubuntu@smithi172.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-04-10T22:38:16.019 INFO:teuthology.run_tasks:Running task console_log... 2024-04-10T22:38:16.094 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fe4ef3c60d0>, signals=[15]) 2024-04-10T22:38:16.095 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-10T22:38:16.101 INFO:teuthology.task.internal:Opening connections... 2024-04-10T22:38:16.101 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi139.front.sepia.ceph.com 2024-04-10T22:38:16.103 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi139.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:38:16.179 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi172.front.sepia.ceph.com 2024-04-10T22:38:16.179 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi172.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:38:16.250 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-10T22:38:16.256 DEBUG:teuthology.orchestra.run.smithi139:> uname -m 2024-04-10T22:38:16.277 INFO:teuthology.orchestra.run.smithi139.stdout:x86_64 2024-04-10T22:38:16.278 DEBUG:teuthology.orchestra.run.smithi139:> cat /etc/os-release 2024-04-10T22:38:16.337 INFO:teuthology.orchestra.run.smithi139.stdout:NAME="Red Hat Enterprise Linux" 2024-04-10T22:38:16.338 INFO:teuthology.orchestra.run.smithi139.stdout:VERSION="8.6 (Ootpa)" 2024-04-10T22:38:16.338 INFO:teuthology.orchestra.run.smithi139.stdout:ID="rhel" 2024-04-10T22:38:16.338 INFO:teuthology.orchestra.run.smithi139.stdout:ID_LIKE="fedora" 2024-04-10T22:38:16.338 INFO:teuthology.orchestra.run.smithi139.stdout:VERSION_ID="8.6" 2024-04-10T22:38:16.338 INFO:teuthology.orchestra.run.smithi139.stdout:PLATFORM_ID="platform:el8" 2024-04-10T22:38:16.338 INFO:teuthology.orchestra.run.smithi139.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2024-04-10T22:38:16.338 INFO:teuthology.orchestra.run.smithi139.stdout:ANSI_COLOR="0;31" 2024-04-10T22:38:16.338 INFO:teuthology.orchestra.run.smithi139.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2024-04-10T22:38:16.338 INFO:teuthology.orchestra.run.smithi139.stdout:HOME_URL="https://www.redhat.com/" 2024-04-10T22:38:16.338 INFO:teuthology.orchestra.run.smithi139.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2024-04-10T22:38:16.338 INFO:teuthology.orchestra.run.smithi139.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-10T22:38:16.338 INFO:teuthology.orchestra.run.smithi139.stdout: 2024-04-10T22:38:16.338 INFO:teuthology.orchestra.run.smithi139.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2024-04-10T22:38:16.339 INFO:teuthology.orchestra.run.smithi139.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2024-04-10T22:38:16.339 INFO:teuthology.orchestra.run.smithi139.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2024-04-10T22:38:16.339 INFO:teuthology.orchestra.run.smithi139.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2024-04-10T22:38:16.339 INFO:teuthology.lock.ops:Updating smithi139.front.sepia.ceph.com on lock server 2024-04-10T22:38:16.362 DEBUG:teuthology.orchestra.run.smithi172:> uname -m 2024-04-10T22:38:16.383 INFO:teuthology.orchestra.run.smithi172.stdout:x86_64 2024-04-10T22:38:16.384 DEBUG:teuthology.orchestra.run.smithi172:> cat /etc/os-release 2024-04-10T22:38:16.443 INFO:teuthology.orchestra.run.smithi172.stdout:NAME="Red Hat Enterprise Linux" 2024-04-10T22:38:16.443 INFO:teuthology.orchestra.run.smithi172.stdout:VERSION="8.6 (Ootpa)" 2024-04-10T22:38:16.443 INFO:teuthology.orchestra.run.smithi172.stdout:ID="rhel" 2024-04-10T22:38:16.443 INFO:teuthology.orchestra.run.smithi172.stdout:ID_LIKE="fedora" 2024-04-10T22:38:16.443 INFO:teuthology.orchestra.run.smithi172.stdout:VERSION_ID="8.6" 2024-04-10T22:38:16.443 INFO:teuthology.orchestra.run.smithi172.stdout:PLATFORM_ID="platform:el8" 2024-04-10T22:38:16.443 INFO:teuthology.orchestra.run.smithi172.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2024-04-10T22:38:16.443 INFO:teuthology.orchestra.run.smithi172.stdout:ANSI_COLOR="0;31" 2024-04-10T22:38:16.443 INFO:teuthology.orchestra.run.smithi172.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2024-04-10T22:38:16.444 INFO:teuthology.orchestra.run.smithi172.stdout:HOME_URL="https://www.redhat.com/" 2024-04-10T22:38:16.444 INFO:teuthology.orchestra.run.smithi172.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2024-04-10T22:38:16.444 INFO:teuthology.orchestra.run.smithi172.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-10T22:38:16.444 INFO:teuthology.orchestra.run.smithi172.stdout: 2024-04-10T22:38:16.444 INFO:teuthology.orchestra.run.smithi172.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2024-04-10T22:38:16.444 INFO:teuthology.orchestra.run.smithi172.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2024-04-10T22:38:16.444 INFO:teuthology.orchestra.run.smithi172.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2024-04-10T22:38:16.444 INFO:teuthology.orchestra.run.smithi172.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2024-04-10T22:38:16.444 INFO:teuthology.lock.ops:Updating smithi172.front.sepia.ceph.com on lock server 2024-04-10T22:38:16.462 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-10T22:38:16.471 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-10T22:38:16.477 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-10T22:38:16.477 DEBUG:teuthology.orchestra.run.smithi139:> test '!' -e /home/ubuntu/cephtest 2024-04-10T22:38:16.480 DEBUG:teuthology.orchestra.run.smithi172:> test '!' -e /home/ubuntu/cephtest 2024-04-10T22:38:16.504 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-10T22:38:16.510 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-10T22:38:16.510 DEBUG:teuthology.orchestra.run.smithi139:> test -z $(ls -A /var/lib/ceph) 2024-04-10T22:38:16.540 DEBUG:teuthology.orchestra.run.smithi172:> test -z $(ls -A /var/lib/ceph) 2024-04-10T22:38:16.575 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-10T22:38:16.632 INFO:teuthology.run_tasks:Running task kernel... 2024-04-10T22:38:16.646 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-04-10T22:38:16.646 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-04-10T22:38:16.646 DEBUG:teuthology.orchestra.run.smithi139:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-10T22:38:16.647 DEBUG:teuthology.orchestra.run.smithi172:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-10T22:38:16.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:38:16.665 DEBUG:teuthology.orchestra.run.smithi172:> uname -r 2024-04-10T22:38:16.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:38:16.666 DEBUG:teuthology.orchestra.run.smithi139:> uname -r 2024-04-10T22:38:16.724 INFO:teuthology.orchestra.run.smithi172.stdout:4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:38:16.724 INFO:teuthology.task.kernel:Running kernel on smithi172: 4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:38:16.724 DEBUG:teuthology.orchestra.run.smithi172:> sudo yum install -y kernel 2024-04-10T22:38:16.726 INFO:teuthology.orchestra.run.smithi139.stdout:4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:38:16.726 INFO:teuthology.task.kernel:Running kernel on smithi139: 4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:38:16.726 DEBUG:teuthology.orchestra.run.smithi139:> sudo yum install -y kernel 2024-04-10T22:38:18.271 INFO:teuthology.orchestra.run.smithi172.stdout:Updating Subscription Management repositories. 2024-04-10T22:38:18.271 INFO:teuthology.orchestra.run.smithi172.stdout:Unable to read consumer identity 2024-04-10T22:38:18.374 INFO:teuthology.orchestra.run.smithi139.stdout:Updating Subscription Management repositories. 2024-04-10T22:38:18.374 INFO:teuthology.orchestra.run.smithi139.stdout:Unable to read consumer identity 2024-04-10T22:38:18.382 INFO:teuthology.orchestra.run.smithi172.stdout: 2024-04-10T22:38:18.382 INFO:teuthology.orchestra.run.smithi172.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:38:18.382 INFO:teuthology.orchestra.run.smithi172.stdout: 2024-04-10T22:38:18.478 INFO:teuthology.orchestra.run.smithi139.stdout: 2024-04-10T22:38:18.478 INFO:teuthology.orchestra.run.smithi139.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:38:18.478 INFO:teuthology.orchestra.run.smithi139.stdout: 2024-04-10T22:39:05.146 INFO:teuthology.orchestra.run.smithi139.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:45 2024-04-10T22:39:05.147 INFO:teuthology.orchestra.run.smithi139.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:39:05.147 INFO:teuthology.orchestra.run.smithi139.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:39:05.147 INFO:teuthology.orchestra.run.smithi139.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:39:05.202 INFO:teuthology.orchestra.run.smithi172.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:45 2024-04-10T22:39:05.203 INFO:teuthology.orchestra.run.smithi172.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:39:05.203 INFO:teuthology.orchestra.run.smithi172.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:39:05.203 INFO:teuthology.orchestra.run.smithi172.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:39:06.895 INFO:teuthology.orchestra.run.smithi139.stdout:Extra Packages for Enterprise Linux 9.3 MB/s | 16 MB 00:01 2024-04-10T22:39:09.768 INFO:teuthology.orchestra.run.smithi172.stdout:Extra Packages for Enterprise Linux 3.6 MB/s | 16 MB 00:04 2024-04-10T22:39:10.445 INFO:teuthology.orchestra.run.smithi139.stdout:lab-extras 400 kB/s | 24 kB 00:00 2024-04-10T22:39:10.519 INFO:teuthology.orchestra.run.smithi139.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:39:11.172 INFO:teuthology.orchestra.run.smithi139.stdout:Package kernel-4.18.0-513.9.1.el8_9.x86_64 is already installed. 2024-04-10T22:39:11.173 INFO:teuthology.orchestra.run.smithi139.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2024-04-10T22:39:11.256 INFO:teuthology.orchestra.run.smithi139.stdout:Dependencies resolved. 2024-04-10T22:39:11.257 INFO:teuthology.orchestra.run.smithi139.stdout:Nothing to do. 2024-04-10T22:39:11.257 INFO:teuthology.orchestra.run.smithi139.stdout:Complete! 2024-04-10T22:39:11.390 DEBUG:teuthology.orchestra.run.smithi139:> echo no | sudo yum reinstall kernel || true 2024-04-10T22:39:11.726 INFO:teuthology.orchestra.run.smithi139.stdout:Updating Subscription Management repositories. 2024-04-10T22:39:11.727 INFO:teuthology.orchestra.run.smithi139.stdout:Unable to read consumer identity 2024-04-10T22:39:11.778 INFO:teuthology.orchestra.run.smithi139.stdout: 2024-04-10T22:39:11.778 INFO:teuthology.orchestra.run.smithi139.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:39:11.778 INFO:teuthology.orchestra.run.smithi139.stdout: 2024-04-10T22:39:13.315 INFO:teuthology.orchestra.run.smithi172.stdout:lab-extras 391 kB/s | 24 kB 00:00 2024-04-10T22:39:13.390 INFO:teuthology.orchestra.run.smithi172.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:39:14.026 INFO:teuthology.orchestra.run.smithi172.stdout:Package kernel-4.18.0-513.9.1.el8_9.x86_64 is already installed. 2024-04-10T22:39:14.026 INFO:teuthology.orchestra.run.smithi172.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2024-04-10T22:39:14.126 INFO:teuthology.orchestra.run.smithi172.stdout:Dependencies resolved. 2024-04-10T22:39:14.128 INFO:teuthology.orchestra.run.smithi172.stdout:Nothing to do. 2024-04-10T22:39:14.128 INFO:teuthology.orchestra.run.smithi172.stdout:Complete! 2024-04-10T22:39:14.275 DEBUG:teuthology.orchestra.run.smithi172:> echo no | sudo yum reinstall kernel || true 2024-04-10T22:39:14.613 INFO:teuthology.orchestra.run.smithi172.stdout:Updating Subscription Management repositories. 2024-04-10T22:39:14.613 INFO:teuthology.orchestra.run.smithi172.stdout:Unable to read consumer identity 2024-04-10T22:39:14.669 INFO:teuthology.orchestra.run.smithi172.stdout: 2024-04-10T22:39:14.670 INFO:teuthology.orchestra.run.smithi172.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:39:14.670 INFO:teuthology.orchestra.run.smithi172.stdout: 2024-04-10T22:39:55.457 INFO:teuthology.orchestra.run.smithi139.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:43 2024-04-10T22:39:55.458 INFO:teuthology.orchestra.run.smithi139.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:39:55.458 INFO:teuthology.orchestra.run.smithi139.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:39:55.458 INFO:teuthology.orchestra.run.smithi139.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:39:55.612 INFO:teuthology.orchestra.run.smithi139.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:39:55.612 INFO:teuthology.orchestra.run.smithi139.stdout:Last metadata expiration check: 0:00:45 ago on Wed 10 Apr 2024 10:39:10 PM UTC. 2024-04-10T22:39:55.712 INFO:teuthology.orchestra.run.smithi139.stdout:Installed package kernel-4.18.0-513.9.1.el8_9.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available. 2024-04-10T22:39:55.712 INFO:teuthology.orchestra.run.smithi139.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2024-04-10T22:39:55.763 INFO:teuthology.orchestra.run.smithi139.stderr:Error: No packages marked for reinstall. 2024-04-10T22:39:55.809 DEBUG:teuthology.orchestra.run.smithi139:> sudo yum reinstall -y kernel || true 2024-04-10T22:39:56.144 INFO:teuthology.orchestra.run.smithi139.stdout:Updating Subscription Management repositories. 2024-04-10T22:39:56.144 INFO:teuthology.orchestra.run.smithi139.stdout:Unable to read consumer identity 2024-04-10T22:39:56.215 INFO:teuthology.orchestra.run.smithi139.stdout: 2024-04-10T22:39:56.215 INFO:teuthology.orchestra.run.smithi139.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:39:56.215 INFO:teuthology.orchestra.run.smithi139.stdout: 2024-04-10T22:40:05.556 INFO:teuthology.orchestra.run.smithi172.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:50 2024-04-10T22:40:05.557 INFO:teuthology.orchestra.run.smithi172.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:40:05.557 INFO:teuthology.orchestra.run.smithi172.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:40:05.557 INFO:teuthology.orchestra.run.smithi172.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:40:05.711 INFO:teuthology.orchestra.run.smithi172.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:40:05.711 INFO:teuthology.orchestra.run.smithi172.stdout:Last metadata expiration check: 0:00:52 ago on Wed 10 Apr 2024 10:39:13 PM UTC. 2024-04-10T22:40:05.797 INFO:teuthology.orchestra.run.smithi172.stdout:Installed package kernel-4.18.0-513.9.1.el8_9.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available. 2024-04-10T22:40:05.797 INFO:teuthology.orchestra.run.smithi172.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2024-04-10T22:40:05.843 INFO:teuthology.orchestra.run.smithi172.stderr:Error: No packages marked for reinstall. 2024-04-10T22:40:05.891 DEBUG:teuthology.orchestra.run.smithi172:> sudo yum reinstall -y kernel || true 2024-04-10T22:40:06.230 INFO:teuthology.orchestra.run.smithi172.stdout:Updating Subscription Management repositories. 2024-04-10T22:40:06.231 INFO:teuthology.orchestra.run.smithi172.stdout:Unable to read consumer identity 2024-04-10T22:40:06.299 INFO:teuthology.orchestra.run.smithi172.stdout: 2024-04-10T22:40:06.299 INFO:teuthology.orchestra.run.smithi172.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:40:06.299 INFO:teuthology.orchestra.run.smithi172.stdout: 2024-04-10T22:40:35.884 INFO:teuthology.orchestra.run.smithi139.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:39 2024-04-10T22:40:35.902 INFO:teuthology.orchestra.run.smithi139.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:40:35.903 INFO:teuthology.orchestra.run.smithi139.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:40:35.903 INFO:teuthology.orchestra.run.smithi139.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:40:36.030 INFO:teuthology.orchestra.run.smithi139.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:40:36.030 INFO:teuthology.orchestra.run.smithi139.stdout:Last metadata expiration check: 0:01:26 ago on Wed 10 Apr 2024 10:39:10 PM UTC. 2024-04-10T22:40:36.093 INFO:teuthology.orchestra.run.smithi139.stdout:Installed package kernel-4.18.0-513.9.1.el8_9.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available. 2024-04-10T22:40:36.093 INFO:teuthology.orchestra.run.smithi139.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2024-04-10T22:40:36.131 INFO:teuthology.orchestra.run.smithi139.stderr:Error: No packages marked for reinstall. 2024-04-10T22:40:36.178 DEBUG:teuthology.orchestra.run.smithi139:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:40:36.332 INFO:teuthology.orchestra.run.smithi139.stdout:kernel-4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:40:36.332 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:40:36.332 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-10T22:40:36.332 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-04-10T22:40:36.332 DEBUG:teuthology.orchestra.run.smithi139:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-10T22:40:36.366 INFO:teuthology.orchestra.run.smithi139.stdout:ttyS1 2024-04-10T22:40:36.390 DEBUG:teuthology.parallel:result is None 2024-04-10T22:40:45.967 INFO:teuthology.orchestra.run.smithi172.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:39 2024-04-10T22:40:45.968 INFO:teuthology.orchestra.run.smithi172.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:40:45.968 INFO:teuthology.orchestra.run.smithi172.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:40:45.968 INFO:teuthology.orchestra.run.smithi172.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:40:46.112 INFO:teuthology.orchestra.run.smithi172.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:40:46.113 INFO:teuthology.orchestra.run.smithi172.stdout:Last metadata expiration check: 0:01:33 ago on Wed 10 Apr 2024 10:39:13 PM UTC. 2024-04-10T22:40:46.180 INFO:teuthology.orchestra.run.smithi172.stdout:Installed package kernel-4.18.0-513.9.1.el8_9.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available. 2024-04-10T22:40:46.181 INFO:teuthology.orchestra.run.smithi172.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2024-04-10T22:40:46.213 INFO:teuthology.orchestra.run.smithi172.stderr:Error: No packages marked for reinstall. 2024-04-10T22:40:46.259 DEBUG:teuthology.orchestra.run.smithi172:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:40:46.416 INFO:teuthology.orchestra.run.smithi172.stdout:kernel-4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:40:46.416 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:40:46.417 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-10T22:40:46.417 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-04-10T22:40:46.417 DEBUG:teuthology.orchestra.run.smithi172:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-10T22:40:46.453 INFO:teuthology.orchestra.run.smithi172.stdout:ttyS1 2024-04-10T22:40:46.467 DEBUG:teuthology.parallel:result is None 2024-04-10T22:40:46.467 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-10T22:40:46.479 INFO:teuthology.task.internal:Creating test directory... 2024-04-10T22:40:46.479 DEBUG:teuthology.orchestra.run.smithi139:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-10T22:40:46.482 DEBUG:teuthology.orchestra.run.smithi172:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-10T22:40:46.524 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-10T22:40:46.531 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-10T22:40:46.541 INFO:teuthology.task.internal:Creating archive directory... 2024-04-10T22:40:46.541 DEBUG:teuthology.orchestra.run.smithi139:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-10T22:40:46.543 DEBUG:teuthology.orchestra.run.smithi172:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-10T22:40:46.598 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-10T22:40:46.605 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-10T22:40:46.605 DEBUG:teuthology.orchestra.run.smithi139:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-10T22:40:46.609 DEBUG:teuthology.orchestra.run.smithi172:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-10T22:40:46.646 INFO:teuthology.orchestra.run.smithi139.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:40:46.658 INFO:teuthology.orchestra.run.smithi139.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:40:46.664 INFO:teuthology.orchestra.run.smithi172.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:40:46.675 INFO:teuthology.orchestra.run.smithi172.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:40:46.676 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-10T22:40:46.684 INFO:teuthology.task.internal:Configuring sudo... 2024-04-10T22:40:46.684 DEBUG:teuthology.orchestra.run.smithi139:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-10T22:40:46.701 DEBUG:teuthology.orchestra.run.smithi172:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-10T22:40:46.743 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-10T22:40:46.752 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-10T22:40:46.752 DEBUG:teuthology.orchestra.run.smithi139:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-10T22:40:46.771 DEBUG:teuthology.orchestra.run.smithi172:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-10T22:40:46.801 DEBUG:teuthology.orchestra.run.smithi139:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:40:46.874 DEBUG:teuthology.orchestra.run.smithi139:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:40:46.933 DEBUG:teuthology.orchestra.run.smithi139:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:40:47.018 DEBUG:teuthology.orchestra.run.smithi139:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:40:47.060 DEBUG:teuthology.orchestra.run.smithi139:> set -ex 2024-04-10T22:40:47.060 DEBUG:teuthology.orchestra.run.smithi139:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-10T22:40:47.127 DEBUG:teuthology.orchestra.run.smithi172:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:40:47.169 DEBUG:teuthology.orchestra.run.smithi172:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:40:47.235 DEBUG:teuthology.orchestra.run.smithi172:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:40:47.277 DEBUG:teuthology.orchestra.run.smithi172:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:40:47.320 DEBUG:teuthology.orchestra.run.smithi172:> set -ex 2024-04-10T22:40:47.321 DEBUG:teuthology.orchestra.run.smithi172:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-10T22:40:47.387 DEBUG:teuthology.orchestra.run.smithi139:> sudo service rsyslog restart 2024-04-10T22:40:47.389 DEBUG:teuthology.orchestra.run.smithi172:> sudo service rsyslog restart 2024-04-10T22:40:47.437 INFO:teuthology.orchestra.run.smithi139.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:40:47.549 INFO:teuthology.orchestra.run.smithi172.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:40:47.720 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-10T22:40:47.728 INFO:teuthology.task.internal:Starting timer... 2024-04-10T22:40:47.728 INFO:teuthology.run_tasks:Running task pcp... 2024-04-10T22:40:47.744 INFO:teuthology.run_tasks:Running task selinux... 2024-04-10T22:40:47.752 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-04-10T22:40:47.752 DEBUG:teuthology.orchestra.run.smithi139:> sudo service auditd rotate 2024-04-10T22:40:47.828 INFO:teuthology.orchestra.run.smithi139.stdout:Rotating logs: 2024-04-10T22:40:47.830 DEBUG:teuthology.orchestra.run.smithi172:> sudo service auditd rotate 2024-04-10T22:40:47.906 INFO:teuthology.orchestra.run.smithi172.stdout:Rotating logs: 2024-04-10T22:40:47.908 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-10T22:40:47.908 DEBUG:teuthology.orchestra.run.smithi139:> /usr/sbin/getenforce 2024-04-10T22:40:47.932 INFO:teuthology.orchestra.run.smithi139.stdout:Permissive 2024-04-10T22:40:47.932 DEBUG:teuthology.orchestra.run.smithi172:> /usr/sbin/getenforce 2024-04-10T22:40:47.959 INFO:teuthology.orchestra.run.smithi172.stdout:Permissive 2024-04-10T22:40:47.959 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi139.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi172.front.sepia.ceph.com': 'permissive'} 2024-04-10T22:40:47.959 DEBUG:teuthology.orchestra.run.smithi139:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:40:47.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:40:48.000 DEBUG:teuthology.orchestra.run.smithi172:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:40:48.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:40:48.026 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-10T22:40:48.027 DEBUG:teuthology.orchestra.run.smithi139:> sudo /usr/sbin/setenforce permissive 2024-04-10T22:40:48.079 DEBUG:teuthology.orchestra.run.smithi172:> sudo /usr/sbin/setenforce permissive 2024-04-10T22:40:48.107 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-10T22:40:48.117 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-10T22:40:48.134 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-10T22:40:48.135 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi139.front.sepia.ceph.com,smithi172.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-10T22:57:30.566 INFO:teuthology.task.ansible:Archiving ansible failure log at: /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650176/ansible_failures.yaml 2024-04-10T22:57:30.568 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/__init__.py", line 123, in __enter__ self.begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 508, in begin super(CephLab, self).begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 342, in begin self.execute_playbook() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 374, in execute_playbook self._handle_failure(command, status) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 402, in _handle_failure raise AnsibleFailedError(failures) teuthology.exceptions.AnsibleFailedError: [',', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at'] 2024-04-10T22:57:30.801 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=58e8d7b9693040ea9273d55e05174493 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/__init__.py", line 123, in __enter__ self.begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 508, in begin super(CephLab, self).begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 342, in begin self.execute_playbook() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 374, in execute_playbook self._handle_failure(command, status) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 402, in _handle_failure raise AnsibleFailedError(failures) teuthology.exceptions.AnsibleFailedError: [',', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at'] 2024-04-10T22:57:30.803 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-10T22:57:30.812 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-10T22:57:30.812 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-10T22:57:30.820 DEBUG:teuthology.orchestra.run.smithi139:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-10T22:57:30.907 DEBUG:teuthology.orchestra.run.smithi172:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-10T22:57:30.973 DEBUG:teuthology.orchestra.run.smithi139:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:57:31.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:57:31.006 DEBUG:teuthology.orchestra.run.smithi172:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:57:31.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:57:31.037 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-10T22:57:31.047 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-10T22:57:31.054 INFO:teuthology.task.internal:Duration was 1003.326187 seconds 2024-04-10T22:57:31.054 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-10T22:57:31.062 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-10T22:57:31.063 DEBUG:teuthology.orchestra.run.smithi139:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-10T22:57:31.065 DEBUG:teuthology.orchestra.run.smithi172:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-10T22:57:31.114 INFO:teuthology.orchestra.run.smithi139.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:57:31.130 INFO:teuthology.orchestra.run.smithi172.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:57:31.445 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-10T22:57:31.445 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi139.front.sepia.ceph.com 2024-04-10T22:57:31.445 DEBUG:teuthology.orchestra.run.smithi139:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-10T22:57:31.469 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi172.front.sepia.ceph.com 2024-04-10T22:57:31.469 DEBUG:teuthology.orchestra.run.smithi172:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-10T22:57:31.491 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-10T22:57:31.492 DEBUG:teuthology.orchestra.run.smithi139:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-10T22:57:31.511 DEBUG:teuthology.orchestra.run.smithi172:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-10T22:57:31.574 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-10T22:57:31.574 DEBUG:teuthology.orchestra.run.smithi139:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-10T22:57:31.594 DEBUG:teuthology.orchestra.run.smithi172:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-10T22:57:31.681 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-10T22:57:31.689 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-10T22:57:31.690 DEBUG:teuthology.orchestra.run.smithi139:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-10T22:57:31.717 DEBUG:teuthology.orchestra.run.smithi172:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-10T22:57:31.744 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-10T22:57:31.753 DEBUG:teuthology.orchestra.run.smithi139:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-10T22:57:31.759 DEBUG:teuthology.orchestra.run.smithi172:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-10T22:57:31.783 INFO:teuthology.orchestra.run.smithi139.stdout:kernel.core_pattern = core 2024-04-10T22:57:31.811 INFO:teuthology.orchestra.run.smithi172.stdout:kernel.core_pattern = core 2024-04-10T22:57:31.842 DEBUG:teuthology.orchestra.run.smithi139:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-10T22:57:31.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:57:31.884 DEBUG:teuthology.orchestra.run.smithi172:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-10T22:57:31.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:57:31.900 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-10T22:57:31.910 INFO:teuthology.task.internal:Transferring archived files... 2024-04-10T22:57:31.911 DEBUG:teuthology.misc:Transferring archived files from smithi139:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650176/remote/smithi139 2024-04-10T22:57:31.912 DEBUG:teuthology.orchestra.run.smithi139:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-10T22:57:31.973 DEBUG:teuthology.misc:Transferring archived files from smithi172:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650176/remote/smithi172 2024-04-10T22:57:31.973 DEBUG:teuthology.orchestra.run.smithi172:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-10T22:57:32.025 INFO:teuthology.task.internal:Removing archive directory... 2024-04-10T22:57:32.025 DEBUG:teuthology.orchestra.run.smithi139:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-10T22:57:32.028 DEBUG:teuthology.orchestra.run.smithi172:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-10T22:57:32.077 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-10T22:57:32.089 INFO:teuthology.task.internal:Not uploading archives. 2024-04-10T22:57:32.089 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-10T22:57:32.097 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-10T22:57:32.098 DEBUG:teuthology.orchestra.run.smithi139:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-10T22:57:32.100 DEBUG:teuthology.orchestra.run.smithi172:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-10T22:57:32.114 INFO:teuthology.orchestra.run.smithi139.stdout: 262171 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 10 22:57 /home/ubuntu/cephtest 2024-04-10T22:57:32.134 INFO:teuthology.orchestra.run.smithi172.stdout: 262171 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 10 22:57 /home/ubuntu/cephtest 2024-04-10T22:57:32.136 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-10T22:57:32.146 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-10T22:57:32.165 INFO:teuthology.nuke:Checking targets against current locks 2024-04-10T22:57:32.204 DEBUG:teuthology.nuke:shortname: smithi139 2024-04-10T22:57:32.204 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T22:57:32.250 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi139.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650176', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:13.867513', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNdCv3N7o0KROGjUFTqcHTfLRaICJWZtjVlBi4CcJyOx2HmXG3vfT/4YBtgPZvNdPxnRX5uZxD3MfQwA8ney8FQ='} 2024-04-10T22:57:32.265 DEBUG:teuthology.nuke:shortname: smithi172 2024-04-10T22:57:32.265 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T22:57:32.314 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi172.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650176', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:13.868841', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBP9QiZiA/cNy65vCOuDm4Qc0T042szbUGjKOAPrzFpdTD0xwkuJp4Nb5IIk6yFV+VkmLtTMRNlu1gagTgtie8n0='} 2024-04-10T22:57:32.372 INFO:teuthology.orchestra.console.smithi139:Power off 2024-04-10T22:57:32.372 DEBUG:teuthology.orchestra.console.smithi139:pexpect command: ipmitool -H smithi139.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-10T22:57:32.408 INFO:teuthology.orchestra.console.smithi172:Power off 2024-04-10T22:57:32.408 DEBUG:teuthology.orchestra.console.smithi172:pexpect command: ipmitool -H smithi172.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-10T22:57:32.427 DEBUG:teuthology.orchestra.console.smithi139:power off output: Chassis Power Control: Down/Off 2024-04-10T22:57:32.427 DEBUG:teuthology.orchestra.console.smithi139:pexpect command: ipmitool -H smithi139.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:57:32.460 DEBUG:teuthology.orchestra.console.smithi172:power off output: Chassis Power Control: Down/Off 2024-04-10T22:57:32.460 DEBUG:teuthology.orchestra.console.smithi172:pexpect command: ipmitool -H smithi172.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:57:32.481 DEBUG:teuthology.orchestra.console.smithi139:check power output: Chassis Power is on 2024-04-10T22:57:32.512 DEBUG:teuthology.orchestra.console.smithi172:check power output: Chassis Power is on 2024-04-10T22:57:36.482 DEBUG:teuthology.orchestra.console.smithi139:pexpect command: ipmitool -H smithi139.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:57:36.512 DEBUG:teuthology.orchestra.console.smithi172:pexpect command: ipmitool -H smithi172.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:57:36.593 DEBUG:teuthology.orchestra.console.smithi139:check power output: Chassis Power is on 2024-04-10T22:57:36.624 DEBUG:teuthology.orchestra.console.smithi172:check power output: Chassis Power is on 2024-04-10T22:57:40.597 DEBUG:teuthology.orchestra.console.smithi139:pexpect command: ipmitool -H smithi139.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:57:40.624 DEBUG:teuthology.orchestra.console.smithi172:pexpect command: ipmitool -H smithi172.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:57:40.709 DEBUG:teuthology.orchestra.console.smithi139:check power output: Chassis Power is on 2024-04-10T22:57:40.733 DEBUG:teuthology.orchestra.console.smithi172:check power output: Chassis Power is off 2024-04-10T22:57:40.834 INFO:teuthology.orchestra.console.smithi172:Power off completed 2024-04-10T22:57:44.713 DEBUG:teuthology.orchestra.console.smithi139:pexpect command: ipmitool -H smithi139.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:57:44.823 DEBUG:teuthology.orchestra.console.smithi139:check power output: Chassis Power is off 2024-04-10T22:57:44.924 INFO:teuthology.orchestra.console.smithi139:Power off completed 2024-04-10T22:57:45.030 INFO:teuthology.run:Summary data: description: upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_rhel8 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} duration: 1003.3261871337891 failure_reason: '['','', ''Bugzilla. In case of problems, contact the owner of this repository.'', ''Enabling a Copr repository. Please note that this repository is not part'', ''Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable'', ''Please do not file bug reports about these packages in Fedora'', ''The Fedora Project does not exercise any power over the contents of'', ''and packages are not held to any quality or security level.'', ''of the main distribution, and quality may vary.'', ''this repository beyond the rules outlined in the Copr FAQ at'']' owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=58e8d7b9693040ea9273d55e05174493 status: dead success: false 2024-04-10T22:57:45.030 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-10T22:57:45.126 INFO:teuthology.run:DEAD