2024-04-10T22:42:14.229 INFO:root:teuthology version: 0.0.1.dev267+g6c63784 2024-04-10T22:42:14.230 DEBUG:teuthology.run:Teuthology command: teuthology --archive /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650200 --name teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi --description upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} --verbose --owner scheduled_teuthology@teuthology -- /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650200/orig.config.yaml 2024-04-10T22:42:14.290 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-10T22:42:14.383 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650200 branch: reef description: upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} email: ceph-qa@ceph.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' first_in_suite: false job_id: '7650200' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run basic import/export cli tests for rbd ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: rhel os_version: '8.6' overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: reef sha1: 4f70a9a90881107996c8b7a3e89ba8b32624713c owner: scheduled_teuthology@teuthology priority: 830 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 4604 sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: pacific clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 0/1 suite: upgrade:pacific-x suite_branch: reef suite_path: /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 4f70a9a90881107996c8b7a3e89ba8b32624713c targets: smithi170.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGow6Yrv/3SIa8dTZzehBAZSTp7Vom8VnBe1mRsIWyTU7qFBkeoWRl0nBGEHNh9po+OmlviiEIOF7YODEaVNCtk= smithi177.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBP/+Z0FDgETmgYxz5Y4KYOcAJPvy3wnijE7AwOT/OJjkYORQTUt4CHjnrW22svEFJGynJYtAQGxxJPjFmiNK6LQ= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools:3.0 --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - install: branch: pacific exclude_packages: - ceph-volume - cephadm: cephadm_branch: pacific cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:pacific - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 6c637841c215537a4502385240412f1966e0faab timestamp: 2024-04-10_01:08:08 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-04-10T22:42:14.383 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa; will attempt to use it 2024-04-10T22:42:14.383 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa/tasks 2024-04-10T22:42:14.384 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-10T22:42:14.385 INFO:teuthology.task.internal:Checking packages... 2024-04-10T22:42:14.410 INFO:teuthology.task.internal:Checking packages for os_type 'rhel', flavor 'default' and ceph hash 'd540ebaca6b131a1dd560e7f69e024b133bbaa42' 2024-04-10T22:42:14.410 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-10T22:42:14.410 INFO:teuthology.packaging:ref: None 2024-04-10T22:42:14.410 INFO:teuthology.packaging:tag: None 2024-04-10T22:42:14.410 INFO:teuthology.packaging:branch: reef 2024-04-10T22:42:14.410 INFO:teuthology.packaging:sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 2024-04-10T22:42:14.410 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef 2024-04-10T22:42:14.774 INFO:teuthology.task.internal:Found packages for ceph version 18.2.2-1189.gc96c8d47 2024-04-10T22:42:14.775 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-10T22:42:14.784 INFO:teuthology.task.internal:no buildpackages task found 2024-04-10T22:42:14.785 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-10T22:42:14.824 INFO:teuthology.task.internal:Saving configuration 2024-04-10T22:42:14.849 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-10T22:42:14.860 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T22:42:14.879 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi170.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650200', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:37:04.460272', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGow6Yrv/3SIa8dTZzehBAZSTp7Vom8VnBe1mRsIWyTU7qFBkeoWRl0nBGEHNh9po+OmlviiEIOF7YODEaVNCtk='} 2024-04-10T22:42:14.899 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi177.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650200', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:37:04.459023', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBP/+Z0FDgETmgYxz5Y4KYOcAJPvy3wnijE7AwOT/OJjkYORQTUt4CHjnrW22svEFJGynJYtAQGxxJPjFmiNK6LQ='} 2024-04-10T22:42:14.899 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-10T22:42:14.910 INFO:teuthology.task.internal:roles: ubuntu@smithi170.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-04-10T22:42:14.910 INFO:teuthology.task.internal:roles: ubuntu@smithi177.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-04-10T22:42:14.910 INFO:teuthology.run_tasks:Running task console_log... 2024-04-10T22:42:15.003 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f68ec5c80d0>, signals=[15]) 2024-04-10T22:42:15.004 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-10T22:42:15.015 INFO:teuthology.task.internal:Opening connections... 2024-04-10T22:42:15.015 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi170.front.sepia.ceph.com 2024-04-10T22:42:15.017 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi170.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:42:15.090 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi177.front.sepia.ceph.com 2024-04-10T22:42:15.091 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi177.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:42:15.163 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-10T22:42:15.170 DEBUG:teuthology.orchestra.run.smithi170:> uname -m 2024-04-10T22:42:15.190 INFO:teuthology.orchestra.run.smithi170.stdout:x86_64 2024-04-10T22:42:15.191 DEBUG:teuthology.orchestra.run.smithi170:> cat /etc/os-release 2024-04-10T22:42:15.250 INFO:teuthology.orchestra.run.smithi170.stdout:NAME="Red Hat Enterprise Linux" 2024-04-10T22:42:15.250 INFO:teuthology.orchestra.run.smithi170.stdout:VERSION="8.6 (Ootpa)" 2024-04-10T22:42:15.250 INFO:teuthology.orchestra.run.smithi170.stdout:ID="rhel" 2024-04-10T22:42:15.250 INFO:teuthology.orchestra.run.smithi170.stdout:ID_LIKE="fedora" 2024-04-10T22:42:15.250 INFO:teuthology.orchestra.run.smithi170.stdout:VERSION_ID="8.6" 2024-04-10T22:42:15.250 INFO:teuthology.orchestra.run.smithi170.stdout:PLATFORM_ID="platform:el8" 2024-04-10T22:42:15.250 INFO:teuthology.orchestra.run.smithi170.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2024-04-10T22:42:15.250 INFO:teuthology.orchestra.run.smithi170.stdout:ANSI_COLOR="0;31" 2024-04-10T22:42:15.250 INFO:teuthology.orchestra.run.smithi170.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2024-04-10T22:42:15.251 INFO:teuthology.orchestra.run.smithi170.stdout:HOME_URL="https://www.redhat.com/" 2024-04-10T22:42:15.251 INFO:teuthology.orchestra.run.smithi170.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2024-04-10T22:42:15.251 INFO:teuthology.orchestra.run.smithi170.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-10T22:42:15.251 INFO:teuthology.orchestra.run.smithi170.stdout: 2024-04-10T22:42:15.251 INFO:teuthology.orchestra.run.smithi170.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2024-04-10T22:42:15.251 INFO:teuthology.orchestra.run.smithi170.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2024-04-10T22:42:15.251 INFO:teuthology.orchestra.run.smithi170.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2024-04-10T22:42:15.251 INFO:teuthology.orchestra.run.smithi170.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2024-04-10T22:42:15.251 INFO:teuthology.lock.ops:Updating smithi170.front.sepia.ceph.com on lock server 2024-04-10T22:42:15.343 DEBUG:teuthology.orchestra.run.smithi177:> uname -m 2024-04-10T22:42:15.366 INFO:teuthology.orchestra.run.smithi177.stdout:x86_64 2024-04-10T22:42:15.366 DEBUG:teuthology.orchestra.run.smithi177:> cat /etc/os-release 2024-04-10T22:42:15.425 INFO:teuthology.orchestra.run.smithi177.stdout:NAME="Red Hat Enterprise Linux" 2024-04-10T22:42:15.425 INFO:teuthology.orchestra.run.smithi177.stdout:VERSION="8.6 (Ootpa)" 2024-04-10T22:42:15.425 INFO:teuthology.orchestra.run.smithi177.stdout:ID="rhel" 2024-04-10T22:42:15.425 INFO:teuthology.orchestra.run.smithi177.stdout:ID_LIKE="fedora" 2024-04-10T22:42:15.425 INFO:teuthology.orchestra.run.smithi177.stdout:VERSION_ID="8.6" 2024-04-10T22:42:15.425 INFO:teuthology.orchestra.run.smithi177.stdout:PLATFORM_ID="platform:el8" 2024-04-10T22:42:15.425 INFO:teuthology.orchestra.run.smithi177.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2024-04-10T22:42:15.425 INFO:teuthology.orchestra.run.smithi177.stdout:ANSI_COLOR="0;31" 2024-04-10T22:42:15.425 INFO:teuthology.orchestra.run.smithi177.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2024-04-10T22:42:15.425 INFO:teuthology.orchestra.run.smithi177.stdout:HOME_URL="https://www.redhat.com/" 2024-04-10T22:42:15.426 INFO:teuthology.orchestra.run.smithi177.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2024-04-10T22:42:15.426 INFO:teuthology.orchestra.run.smithi177.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-10T22:42:15.426 INFO:teuthology.orchestra.run.smithi177.stdout: 2024-04-10T22:42:15.426 INFO:teuthology.orchestra.run.smithi177.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2024-04-10T22:42:15.426 INFO:teuthology.orchestra.run.smithi177.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2024-04-10T22:42:15.426 INFO:teuthology.orchestra.run.smithi177.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2024-04-10T22:42:15.426 INFO:teuthology.orchestra.run.smithi177.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2024-04-10T22:42:15.426 INFO:teuthology.lock.ops:Updating smithi177.front.sepia.ceph.com on lock server 2024-04-10T22:42:15.448 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-10T22:42:15.460 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-10T22:42:15.506 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-10T22:42:15.507 DEBUG:teuthology.orchestra.run.smithi170:> test '!' -e /home/ubuntu/cephtest 2024-04-10T22:42:15.509 DEBUG:teuthology.orchestra.run.smithi177:> test '!' -e /home/ubuntu/cephtest 2024-04-10T22:42:15.528 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-10T22:42:15.571 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-10T22:42:15.571 DEBUG:teuthology.orchestra.run.smithi170:> test -z $(ls -A /var/lib/ceph) 2024-04-10T22:42:15.574 DEBUG:teuthology.orchestra.run.smithi177:> test -z $(ls -A /var/lib/ceph) 2024-04-10T22:42:15.603 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-10T22:42:15.658 INFO:teuthology.run_tasks:Running task kernel... 2024-04-10T22:42:15.674 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-04-10T22:42:15.674 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-04-10T22:42:15.674 DEBUG:teuthology.orchestra.run.smithi170:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-10T22:42:15.675 DEBUG:teuthology.orchestra.run.smithi177:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-10T22:42:15.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:42:15.694 DEBUG:teuthology.orchestra.run.smithi170:> uname -r 2024-04-10T22:42:15.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:42:15.695 DEBUG:teuthology.orchestra.run.smithi177:> uname -r 2024-04-10T22:42:15.755 INFO:teuthology.orchestra.run.smithi170.stdout:4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:42:15.755 INFO:teuthology.orchestra.run.smithi177.stdout:4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:42:15.755 INFO:teuthology.task.kernel:Running kernel on smithi170: 4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:42:15.755 DEBUG:teuthology.orchestra.run.smithi170:> sudo yum install -y kernel 2024-04-10T22:42:15.756 INFO:teuthology.task.kernel:Running kernel on smithi177: 4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:42:15.756 DEBUG:teuthology.orchestra.run.smithi177:> sudo yum install -y kernel 2024-04-10T22:42:17.180 INFO:teuthology.orchestra.run.smithi170.stdout:Updating Subscription Management repositories. 2024-04-10T22:42:17.180 INFO:teuthology.orchestra.run.smithi170.stdout:Unable to read consumer identity 2024-04-10T22:42:17.293 INFO:teuthology.orchestra.run.smithi170.stdout: 2024-04-10T22:42:17.294 INFO:teuthology.orchestra.run.smithi170.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:42:17.294 INFO:teuthology.orchestra.run.smithi170.stdout: 2024-04-10T22:42:17.365 INFO:teuthology.orchestra.run.smithi177.stdout:Updating Subscription Management repositories. 2024-04-10T22:42:17.365 INFO:teuthology.orchestra.run.smithi177.stdout:Unable to read consumer identity 2024-04-10T22:42:17.490 INFO:teuthology.orchestra.run.smithi177.stdout: 2024-04-10T22:42:17.490 INFO:teuthology.orchestra.run.smithi177.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:42:17.490 INFO:teuthology.orchestra.run.smithi177.stdout: 2024-04-10T22:42:57.341 INFO:teuthology.orchestra.run.smithi170.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:38 2024-04-10T22:42:57.341 INFO:teuthology.orchestra.run.smithi170.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:42:57.341 INFO:teuthology.orchestra.run.smithi170.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:42:57.342 INFO:teuthology.orchestra.run.smithi170.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:42:57.403 INFO:teuthology.orchestra.run.smithi177.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:38 2024-04-10T22:42:57.404 INFO:teuthology.orchestra.run.smithi177.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:42:57.404 INFO:teuthology.orchestra.run.smithi177.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:42:57.404 INFO:teuthology.orchestra.run.smithi177.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:42:58.163 INFO:teuthology.orchestra.run.smithi170.stdout:Extra Packages for Enterprise Linux 20 MB/s | 16 MB 00:00 2024-04-10T22:42:58.704 INFO:teuthology.orchestra.run.smithi177.stdout:Extra Packages for Enterprise Linux 13 MB/s | 16 MB 00:01 2024-04-10T22:43:01.736 INFO:teuthology.orchestra.run.smithi170.stdout:lab-extras 418 kB/s | 24 kB 00:00 2024-04-10T22:43:01.811 INFO:teuthology.orchestra.run.smithi170.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:43:02.316 INFO:teuthology.orchestra.run.smithi177.stdout:lab-extras 487 kB/s | 24 kB 00:00 2024-04-10T22:43:02.389 INFO:teuthology.orchestra.run.smithi177.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:43:02.447 INFO:teuthology.orchestra.run.smithi170.stdout:Package kernel-4.18.0-513.9.1.el8_9.x86_64 is already installed. 2024-04-10T22:43:02.447 INFO:teuthology.orchestra.run.smithi170.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2024-04-10T22:43:02.521 INFO:teuthology.orchestra.run.smithi170.stdout:Dependencies resolved. 2024-04-10T22:43:02.522 INFO:teuthology.orchestra.run.smithi170.stdout:Nothing to do. 2024-04-10T22:43:02.522 INFO:teuthology.orchestra.run.smithi170.stdout:Complete! 2024-04-10T22:43:02.649 DEBUG:teuthology.orchestra.run.smithi170:> echo no | sudo yum reinstall kernel || true 2024-04-10T22:43:02.985 INFO:teuthology.orchestra.run.smithi177.stdout:Package kernel-4.18.0-513.9.1.el8_9.x86_64 is already installed. 2024-04-10T22:43:02.985 INFO:teuthology.orchestra.run.smithi177.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2024-04-10T22:43:02.987 INFO:teuthology.orchestra.run.smithi170.stdout:Updating Subscription Management repositories. 2024-04-10T22:43:02.987 INFO:teuthology.orchestra.run.smithi170.stdout:Unable to read consumer identity 2024-04-10T22:43:03.057 INFO:teuthology.orchestra.run.smithi177.stdout:Dependencies resolved. 2024-04-10T22:43:03.058 INFO:teuthology.orchestra.run.smithi177.stdout:Nothing to do. 2024-04-10T22:43:03.058 INFO:teuthology.orchestra.run.smithi177.stdout:Complete! 2024-04-10T22:43:03.089 INFO:teuthology.orchestra.run.smithi170.stdout: 2024-04-10T22:43:03.089 INFO:teuthology.orchestra.run.smithi170.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:43:03.089 INFO:teuthology.orchestra.run.smithi170.stdout: 2024-04-10T22:43:03.176 DEBUG:teuthology.orchestra.run.smithi177:> echo no | sudo yum reinstall kernel || true 2024-04-10T22:43:03.513 INFO:teuthology.orchestra.run.smithi177.stdout:Updating Subscription Management repositories. 2024-04-10T22:43:03.514 INFO:teuthology.orchestra.run.smithi177.stdout:Unable to read consumer identity 2024-04-10T22:43:03.573 INFO:teuthology.orchestra.run.smithi177.stdout: 2024-04-10T22:43:03.573 INFO:teuthology.orchestra.run.smithi177.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:43:03.573 INFO:teuthology.orchestra.run.smithi177.stdout: 2024-04-10T22:43:47.859 INFO:teuthology.orchestra.run.smithi177.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:44 2024-04-10T22:43:47.860 INFO:teuthology.orchestra.run.smithi177.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:43:47.860 INFO:teuthology.orchestra.run.smithi177.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:43:47.860 INFO:teuthology.orchestra.run.smithi177.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:43:47.963 INFO:teuthology.orchestra.run.smithi170.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:44 2024-04-10T22:43:47.964 INFO:teuthology.orchestra.run.smithi170.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:43:47.964 INFO:teuthology.orchestra.run.smithi170.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:43:47.964 INFO:teuthology.orchestra.run.smithi170.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:43:48.016 INFO:teuthology.orchestra.run.smithi177.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:43:48.017 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:00:46 ago on Wed 10 Apr 2024 10:43:02 PM UTC. 2024-04-10T22:43:48.110 INFO:teuthology.orchestra.run.smithi177.stdout:Installed package kernel-4.18.0-513.9.1.el8_9.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available. 2024-04-10T22:43:48.112 INFO:teuthology.orchestra.run.smithi177.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2024-04-10T22:43:48.121 INFO:teuthology.orchestra.run.smithi170.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:43:48.121 INFO:teuthology.orchestra.run.smithi170.stdout:Last metadata expiration check: 0:00:47 ago on Wed 10 Apr 2024 10:43:01 PM UTC. 2024-04-10T22:43:48.149 INFO:teuthology.orchestra.run.smithi177.stderr:Error: No packages marked for reinstall. 2024-04-10T22:43:48.197 DEBUG:teuthology.orchestra.run.smithi177:> sudo yum reinstall -y kernel || true 2024-04-10T22:43:48.240 INFO:teuthology.orchestra.run.smithi170.stdout:Installed package kernel-4.18.0-513.9.1.el8_9.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available. 2024-04-10T22:43:48.243 INFO:teuthology.orchestra.run.smithi170.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2024-04-10T22:43:48.294 INFO:teuthology.orchestra.run.smithi170.stderr:Error: No packages marked for reinstall. 2024-04-10T22:43:48.342 DEBUG:teuthology.orchestra.run.smithi170:> sudo yum reinstall -y kernel || true 2024-04-10T22:43:48.531 INFO:teuthology.orchestra.run.smithi177.stdout:Updating Subscription Management repositories. 2024-04-10T22:43:48.532 INFO:teuthology.orchestra.run.smithi177.stdout:Unable to read consumer identity 2024-04-10T22:43:48.591 INFO:teuthology.orchestra.run.smithi177.stdout: 2024-04-10T22:43:48.591 INFO:teuthology.orchestra.run.smithi177.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:43:48.591 INFO:teuthology.orchestra.run.smithi177.stdout: 2024-04-10T22:43:48.679 INFO:teuthology.orchestra.run.smithi170.stdout:Updating Subscription Management repositories. 2024-04-10T22:43:48.679 INFO:teuthology.orchestra.run.smithi170.stdout:Unable to read consumer identity 2024-04-10T22:43:48.732 INFO:teuthology.orchestra.run.smithi170.stdout: 2024-04-10T22:43:48.732 INFO:teuthology.orchestra.run.smithi170.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:43:48.733 INFO:teuthology.orchestra.run.smithi170.stdout: 2024-04-10T22:44:28.201 INFO:teuthology.orchestra.run.smithi170.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:39 2024-04-10T22:44:28.201 INFO:teuthology.orchestra.run.smithi170.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:44:28.201 INFO:teuthology.orchestra.run.smithi170.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:44:28.202 INFO:teuthology.orchestra.run.smithi170.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:44:28.261 INFO:teuthology.orchestra.run.smithi177.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:39 2024-04-10T22:44:28.262 INFO:teuthology.orchestra.run.smithi177.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:44:28.262 INFO:teuthology.orchestra.run.smithi177.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:44:28.262 INFO:teuthology.orchestra.run.smithi177.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:44:28.351 INFO:teuthology.orchestra.run.smithi170.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:44:28.352 INFO:teuthology.orchestra.run.smithi170.stdout:Last metadata expiration check: 0:01:27 ago on Wed 10 Apr 2024 10:43:01 PM UTC. 2024-04-10T22:44:28.409 INFO:teuthology.orchestra.run.smithi177.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:44:28.409 INFO:teuthology.orchestra.run.smithi177.stdout:Last metadata expiration check: 0:01:26 ago on Wed 10 Apr 2024 10:43:02 PM UTC. 2024-04-10T22:44:28.424 INFO:teuthology.orchestra.run.smithi170.stdout:Installed package kernel-4.18.0-513.9.1.el8_9.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available. 2024-04-10T22:44:28.424 INFO:teuthology.orchestra.run.smithi170.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2024-04-10T22:44:28.462 INFO:teuthology.orchestra.run.smithi170.stderr:Error: No packages marked for reinstall. 2024-04-10T22:44:28.472 INFO:teuthology.orchestra.run.smithi177.stdout:Installed package kernel-4.18.0-513.9.1.el8_9.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available. 2024-04-10T22:44:28.472 INFO:teuthology.orchestra.run.smithi177.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2024-04-10T22:44:28.513 DEBUG:teuthology.orchestra.run.smithi170:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:44:28.514 INFO:teuthology.orchestra.run.smithi177.stderr:Error: No packages marked for reinstall. 2024-04-10T22:44:28.561 DEBUG:teuthology.orchestra.run.smithi177:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:44:28.667 INFO:teuthology.orchestra.run.smithi170.stdout:kernel-4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:44:28.668 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:44:28.668 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-10T22:44:28.668 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-04-10T22:44:28.668 DEBUG:teuthology.orchestra.run.smithi170:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-10T22:44:28.703 INFO:teuthology.orchestra.run.smithi170.stdout:ttyS1 2024-04-10T22:44:28.716 DEBUG:teuthology.parallel:result is None 2024-04-10T22:44:28.737 INFO:teuthology.orchestra.run.smithi177.stdout:kernel-4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:44:28.738 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:44:28.738 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-10T22:44:28.739 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-04-10T22:44:28.739 DEBUG:teuthology.orchestra.run.smithi177:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-10T22:44:28.776 INFO:teuthology.orchestra.run.smithi177.stdout:ttyS1 2024-04-10T22:44:28.788 DEBUG:teuthology.parallel:result is None 2024-04-10T22:44:28.789 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-10T22:44:28.796 INFO:teuthology.task.internal:Creating test directory... 2024-04-10T22:44:28.796 DEBUG:teuthology.orchestra.run.smithi170:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-10T22:44:28.799 DEBUG:teuthology.orchestra.run.smithi177:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-10T22:44:28.846 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-10T22:44:28.852 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-10T22:44:28.889 INFO:teuthology.task.internal:Creating archive directory... 2024-04-10T22:44:28.890 DEBUG:teuthology.orchestra.run.smithi170:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-10T22:44:28.892 DEBUG:teuthology.orchestra.run.smithi177:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-10T22:44:28.925 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-10T22:44:28.964 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-10T22:44:28.964 DEBUG:teuthology.orchestra.run.smithi170:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-10T22:44:28.967 DEBUG:teuthology.orchestra.run.smithi177:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-10T22:44:28.998 INFO:teuthology.orchestra.run.smithi170.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:44:29.000 INFO:teuthology.orchestra.run.smithi177.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:44:29.008 INFO:teuthology.orchestra.run.smithi170.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:44:29.011 INFO:teuthology.orchestra.run.smithi177.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:44:29.012 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-10T22:44:29.019 INFO:teuthology.task.internal:Configuring sudo... 2024-04-10T22:44:29.019 DEBUG:teuthology.orchestra.run.smithi170:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-10T22:44:29.052 DEBUG:teuthology.orchestra.run.smithi177:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-10T22:44:29.079 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-10T22:44:29.088 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-10T22:44:29.088 DEBUG:teuthology.orchestra.run.smithi170:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-10T22:44:29.119 DEBUG:teuthology.orchestra.run.smithi177:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-10T22:44:29.135 DEBUG:teuthology.orchestra.run.smithi170:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:44:29.230 DEBUG:teuthology.orchestra.run.smithi170:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:44:29.268 DEBUG:teuthology.orchestra.run.smithi170:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:44:29.362 DEBUG:teuthology.orchestra.run.smithi170:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:44:29.393 DEBUG:teuthology.orchestra.run.smithi170:> set -ex 2024-04-10T22:44:29.393 DEBUG:teuthology.orchestra.run.smithi170:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-10T22:44:29.460 DEBUG:teuthology.orchestra.run.smithi177:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:44:29.511 DEBUG:teuthology.orchestra.run.smithi177:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:44:29.553 DEBUG:teuthology.orchestra.run.smithi177:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:44:29.658 DEBUG:teuthology.orchestra.run.smithi177:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:44:29.685 DEBUG:teuthology.orchestra.run.smithi177:> set -ex 2024-04-10T22:44:29.685 DEBUG:teuthology.orchestra.run.smithi177:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-10T22:44:29.751 DEBUG:teuthology.orchestra.run.smithi170:> sudo service rsyslog restart 2024-04-10T22:44:29.753 DEBUG:teuthology.orchestra.run.smithi177:> sudo service rsyslog restart 2024-04-10T22:44:29.801 INFO:teuthology.orchestra.run.smithi170.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:44:29.840 INFO:teuthology.orchestra.run.smithi177.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:44:30.301 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-10T22:44:30.310 INFO:teuthology.task.internal:Starting timer... 2024-04-10T22:44:30.310 INFO:teuthology.run_tasks:Running task pcp... 2024-04-10T22:44:30.363 INFO:teuthology.run_tasks:Running task selinux... 2024-04-10T22:44:30.409 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-04-10T22:44:30.409 DEBUG:teuthology.orchestra.run.smithi170:> sudo service auditd rotate 2024-04-10T22:44:30.464 INFO:teuthology.orchestra.run.smithi170.stdout:Rotating logs: 2024-04-10T22:44:30.466 DEBUG:teuthology.orchestra.run.smithi177:> sudo service auditd rotate 2024-04-10T22:44:30.535 INFO:teuthology.orchestra.run.smithi177.stdout:Rotating logs: 2024-04-10T22:44:30.537 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-10T22:44:30.537 DEBUG:teuthology.orchestra.run.smithi170:> /usr/sbin/getenforce 2024-04-10T22:44:30.558 INFO:teuthology.orchestra.run.smithi170.stdout:Permissive 2024-04-10T22:44:30.559 DEBUG:teuthology.orchestra.run.smithi177:> /usr/sbin/getenforce 2024-04-10T22:44:30.582 INFO:teuthology.orchestra.run.smithi177.stdout:Permissive 2024-04-10T22:44:30.583 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi170.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi177.front.sepia.ceph.com': 'permissive'} 2024-04-10T22:44:30.583 DEBUG:teuthology.orchestra.run.smithi170:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:44:30.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:44:30.625 DEBUG:teuthology.orchestra.run.smithi177:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:44:30.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:44:30.652 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-10T22:44:30.652 DEBUG:teuthology.orchestra.run.smithi170:> sudo /usr/sbin/setenforce permissive 2024-04-10T22:44:30.692 DEBUG:teuthology.orchestra.run.smithi177:> sudo /usr/sbin/setenforce permissive 2024-04-10T22:44:30.719 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-10T22:44:30.729 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-10T22:44:30.746 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-10T22:44:30.747 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi170.front.sepia.ceph.com,smithi177.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-10T23:01:29.960 INFO:teuthology.task.ansible:Archiving ansible failure log at: /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650200/ansible_failures.yaml 2024-04-10T23:01:29.962 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/__init__.py", line 123, in __enter__ self.begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 508, in begin super(CephLab, self).begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 342, in begin self.execute_playbook() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 374, in execute_playbook self._handle_failure(command, status) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 402, in _handle_failure raise AnsibleFailedError(failures) teuthology.exceptions.AnsibleFailedError: [',', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at'] 2024-04-10T23:01:30.178 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=bf31817eef7443198fe49b60d4cb6f07 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/__init__.py", line 123, in __enter__ self.begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 508, in begin super(CephLab, self).begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 342, in begin self.execute_playbook() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 374, in execute_playbook self._handle_failure(command, status) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 402, in _handle_failure raise AnsibleFailedError(failures) teuthology.exceptions.AnsibleFailedError: [',', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at'] 2024-04-10T23:01:30.180 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-10T23:01:30.189 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-10T23:01:30.189 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-10T23:01:30.230 DEBUG:teuthology.orchestra.run.smithi170:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-10T23:01:30.313 DEBUG:teuthology.orchestra.run.smithi177:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-10T23:01:30.394 DEBUG:teuthology.orchestra.run.smithi170:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T23:01:30.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T23:01:30.425 DEBUG:teuthology.orchestra.run.smithi177:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T23:01:30.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T23:01:30.456 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-10T23:01:30.466 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-10T23:01:30.489 INFO:teuthology.task.internal:Duration was 1020.178851 seconds 2024-04-10T23:01:30.489 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-10T23:01:30.539 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-10T23:01:30.539 DEBUG:teuthology.orchestra.run.smithi170:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-10T23:01:30.542 DEBUG:teuthology.orchestra.run.smithi177:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-10T23:01:30.589 INFO:teuthology.orchestra.run.smithi170.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T23:01:30.589 INFO:teuthology.orchestra.run.smithi177.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T23:01:30.925 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-10T23:01:30.926 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi170.front.sepia.ceph.com 2024-04-10T23:01:30.926 DEBUG:teuthology.orchestra.run.smithi170:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-10T23:01:30.949 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi177.front.sepia.ceph.com 2024-04-10T23:01:30.949 DEBUG:teuthology.orchestra.run.smithi177:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-10T23:01:30.971 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-10T23:01:30.971 DEBUG:teuthology.orchestra.run.smithi170:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-10T23:01:30.992 DEBUG:teuthology.orchestra.run.smithi177:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-10T23:01:31.053 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-10T23:01:31.053 DEBUG:teuthology.orchestra.run.smithi170:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-10T23:01:31.076 DEBUG:teuthology.orchestra.run.smithi177:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-10T23:01:31.157 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-10T23:01:31.166 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-10T23:01:31.166 DEBUG:teuthology.orchestra.run.smithi170:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-10T23:01:31.193 DEBUG:teuthology.orchestra.run.smithi177:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-10T23:01:31.219 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-10T23:01:31.228 DEBUG:teuthology.orchestra.run.smithi170:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-10T23:01:31.235 DEBUG:teuthology.orchestra.run.smithi177:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-10T23:01:31.259 INFO:teuthology.orchestra.run.smithi170.stdout:kernel.core_pattern = core 2024-04-10T23:01:31.286 INFO:teuthology.orchestra.run.smithi177.stdout:kernel.core_pattern = core 2024-04-10T23:01:31.324 DEBUG:teuthology.orchestra.run.smithi170:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-10T23:01:31.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T23:01:31.348 DEBUG:teuthology.orchestra.run.smithi177:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-10T23:01:31.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T23:01:31.380 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-10T23:01:31.390 INFO:teuthology.task.internal:Transferring archived files... 2024-04-10T23:01:31.391 DEBUG:teuthology.misc:Transferring archived files from smithi170:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650200/remote/smithi170 2024-04-10T23:01:31.392 DEBUG:teuthology.orchestra.run.smithi170:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-10T23:01:31.439 DEBUG:teuthology.misc:Transferring archived files from smithi177:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650200/remote/smithi177 2024-04-10T23:01:31.440 DEBUG:teuthology.orchestra.run.smithi177:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-10T23:01:31.496 INFO:teuthology.task.internal:Removing archive directory... 2024-04-10T23:01:31.496 DEBUG:teuthology.orchestra.run.smithi170:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-10T23:01:31.499 DEBUG:teuthology.orchestra.run.smithi177:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-10T23:01:31.547 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-10T23:01:31.558 INFO:teuthology.task.internal:Not uploading archives. 2024-04-10T23:01:31.558 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-10T23:01:31.582 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-10T23:01:31.582 DEBUG:teuthology.orchestra.run.smithi170:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-10T23:01:31.584 DEBUG:teuthology.orchestra.run.smithi177:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-10T23:01:31.598 INFO:teuthology.orchestra.run.smithi170.stdout: 262171 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 10 23:01 /home/ubuntu/cephtest 2024-04-10T23:01:31.603 INFO:teuthology.orchestra.run.smithi177.stdout: 262171 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 10 23:01 /home/ubuntu/cephtest 2024-04-10T23:01:31.605 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-10T23:01:31.632 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-10T23:01:31.732 INFO:teuthology.nuke:Checking targets against current locks 2024-04-10T23:01:31.773 DEBUG:teuthology.nuke:shortname: smithi170 2024-04-10T23:01:31.773 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T23:01:31.815 DEBUG:teuthology.nuke:shortname: smithi177 2024-04-10T23:01:31.816 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T23:01:31.830 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi170.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650200', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:37:04.460272', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGow6Yrv/3SIa8dTZzehBAZSTp7Vom8VnBe1mRsIWyTU7qFBkeoWRl0nBGEHNh9po+OmlviiEIOF7YODEaVNCtk='} 2024-04-10T23:01:31.864 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi177.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650200', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:37:04.459023', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBP/+Z0FDgETmgYxz5Y4KYOcAJPvy3wnijE7AwOT/OJjkYORQTUt4CHjnrW22svEFJGynJYtAQGxxJPjFmiNK6LQ='} 2024-04-10T23:01:31.918 INFO:teuthology.orchestra.console.smithi170:Power off 2024-04-10T23:01:31.919 DEBUG:teuthology.orchestra.console.smithi170:pexpect command: ipmitool -H smithi170.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-10T23:01:31.966 INFO:teuthology.orchestra.console.smithi177:Power off 2024-04-10T23:01:31.967 DEBUG:teuthology.orchestra.console.smithi177:pexpect command: ipmitool -H smithi177.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-10T23:01:31.976 DEBUG:teuthology.orchestra.console.smithi170:power off output: Chassis Power Control: Down/Off 2024-04-10T23:01:31.977 DEBUG:teuthology.orchestra.console.smithi170:pexpect command: ipmitool -H smithi170.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:01:32.026 DEBUG:teuthology.orchestra.console.smithi177:power off output: Chassis Power Control: Down/Off 2024-04-10T23:01:32.027 DEBUG:teuthology.orchestra.console.smithi177:pexpect command: ipmitool -H smithi177.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:01:32.035 DEBUG:teuthology.orchestra.console.smithi170:check power output: Chassis Power is on 2024-04-10T23:01:32.078 DEBUG:teuthology.orchestra.console.smithi177:check power output: Chassis Power is on 2024-04-10T23:01:36.037 DEBUG:teuthology.orchestra.console.smithi170:pexpect command: ipmitool -H smithi170.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:01:36.079 DEBUG:teuthology.orchestra.console.smithi177:pexpect command: ipmitool -H smithi177.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:01:36.148 DEBUG:teuthology.orchestra.console.smithi170:check power output: Chassis Power is on 2024-04-10T23:01:36.188 DEBUG:teuthology.orchestra.console.smithi177:check power output: Chassis Power is on 2024-04-10T23:01:40.151 DEBUG:teuthology.orchestra.console.smithi170:pexpect command: ipmitool -H smithi170.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:01:40.189 DEBUG:teuthology.orchestra.console.smithi177:pexpect command: ipmitool -H smithi177.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:01:40.261 DEBUG:teuthology.orchestra.console.smithi170:check power output: Chassis Power is off 2024-04-10T23:01:40.300 DEBUG:teuthology.orchestra.console.smithi177:check power output: Chassis Power is off 2024-04-10T23:01:40.363 INFO:teuthology.orchestra.console.smithi170:Power off completed 2024-04-10T23:01:40.401 INFO:teuthology.orchestra.console.smithi177:Power off completed 2024-04-10T23:01:40.506 INFO:teuthology.run:Summary data: description: upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} duration: 1020.1788506507874 failure_reason: '['','', ''Bugzilla. In case of problems, contact the owner of this repository.'', ''Enabling a Copr repository. Please note that this repository is not part'', ''Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable'', ''Please do not file bug reports about these packages in Fedora'', ''The Fedora Project does not exercise any power over the contents of'', ''and packages are not held to any quality or security level.'', ''of the main distribution, and quality may vary.'', ''this repository beyond the rules outlined in the Copr FAQ at'']' owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=bf31817eef7443198fe49b60d4cb6f07 status: dead success: false 2024-04-10T23:01:40.507 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-10T23:01:40.612 INFO:teuthology.run:DEAD