2024-04-10T22:38:59.149 INFO:root:teuthology version: 0.0.1.dev267+g6c63784 2024-04-10T22:38:59.150 DEBUG:teuthology.run:Teuthology command: teuthology --owner scheduled_teuthology@teuthology --name teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi --description upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_rhel8 0-roles 1-start 2-first-half-tasks/readwrite 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} --verbose --archive /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650196 -- /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650196/orig.config.yaml 2024-04-10T22:38:59.206 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-10T22:38:59.295 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650196 branch: reef description: upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_rhel8 0-roles 1-start 2-first-half-tasks/readwrite 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} email: ceph-qa@ceph.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' first_in_suite: false job_id: '7650196' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: rhel os_version: '8.6' overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: reef sha1: 4f70a9a90881107996c8b7a3e89ba8b32624713c owner: scheduled_teuthology@teuthology priority: 830 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 4604 sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: pacific clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 0/1 suite: upgrade:pacific-x suite_branch: reef suite_path: /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 4f70a9a90881107996c8b7a3e89ba8b32624713c targets: smithi142.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBH9asL/GBiwXWLhxPgHH+NR8jEaZdBitJfqCxhsRrgZ+v0tOjf++dShNsuyZGcze5lLUJldJuyMM/AlMugoW01M= smithi150.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNSRi03CHd0+TAOz3vlIezFRx1PaZsdNziFiTMZ6Da27IzlTvod6VGT8wjlDWPOCEtQNKMvd/TRY67kZeNQq1Pk= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools:rhel8 --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - install: branch: pacific exclude_packages: - ceph-volume - cephadm: cephadm_branch: pacific cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:pacific - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 6c637841c215537a4502385240412f1966e0faab timestamp: 2024-04-10_01:08:08 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-04-10T22:38:59.295 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa; will attempt to use it 2024-04-10T22:38:59.296 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa/tasks 2024-04-10T22:38:59.296 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-10T22:38:59.297 INFO:teuthology.task.internal:Checking packages... 2024-04-10T22:38:59.318 INFO:teuthology.task.internal:Checking packages for os_type 'rhel', flavor 'default' and ceph hash 'd540ebaca6b131a1dd560e7f69e024b133bbaa42' 2024-04-10T22:38:59.318 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-10T22:38:59.318 INFO:teuthology.packaging:ref: None 2024-04-10T22:38:59.318 INFO:teuthology.packaging:tag: None 2024-04-10T22:38:59.318 INFO:teuthology.packaging:branch: reef 2024-04-10T22:38:59.318 INFO:teuthology.packaging:sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 2024-04-10T22:38:59.318 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef 2024-04-10T22:38:59.674 INFO:teuthology.task.internal:Found packages for ceph version 18.2.2-1189.gc96c8d47 2024-04-10T22:38:59.675 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-10T22:38:59.682 INFO:teuthology.task.internal:no buildpackages task found 2024-04-10T22:38:59.682 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-10T22:38:59.709 INFO:teuthology.task.internal:Saving configuration 2024-04-10T22:38:59.730 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-10T22:38:59.736 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T22:38:59.763 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi142.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650196', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:22.121620', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBH9asL/GBiwXWLhxPgHH+NR8jEaZdBitJfqCxhsRrgZ+v0tOjf++dShNsuyZGcze5lLUJldJuyMM/AlMugoW01M='} 2024-04-10T22:38:59.786 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi150.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650196', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:22.123101', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNSRi03CHd0+TAOz3vlIezFRx1PaZsdNziFiTMZ6Da27IzlTvod6VGT8wjlDWPOCEtQNKMvd/TRY67kZeNQq1Pk='} 2024-04-10T22:38:59.786 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-10T22:38:59.793 INFO:teuthology.task.internal:roles: ubuntu@smithi142.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-04-10T22:38:59.793 INFO:teuthology.task.internal:roles: ubuntu@smithi150.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-04-10T22:38:59.793 INFO:teuthology.run_tasks:Running task console_log... 2024-04-10T22:38:59.876 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7febc92190d0>, signals=[15]) 2024-04-10T22:38:59.876 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-10T22:38:59.882 INFO:teuthology.task.internal:Opening connections... 2024-04-10T22:38:59.883 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi142.front.sepia.ceph.com 2024-04-10T22:38:59.884 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi142.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:38:59.958 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi150.front.sepia.ceph.com 2024-04-10T22:38:59.959 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi150.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:39:00.033 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-10T22:39:00.039 DEBUG:teuthology.orchestra.run.smithi142:> uname -m 2024-04-10T22:39:00.061 INFO:teuthology.orchestra.run.smithi142.stdout:x86_64 2024-04-10T22:39:00.061 DEBUG:teuthology.orchestra.run.smithi142:> cat /etc/os-release 2024-04-10T22:39:00.120 INFO:teuthology.orchestra.run.smithi142.stdout:NAME="Red Hat Enterprise Linux" 2024-04-10T22:39:00.121 INFO:teuthology.orchestra.run.smithi142.stdout:VERSION="8.6 (Ootpa)" 2024-04-10T22:39:00.121 INFO:teuthology.orchestra.run.smithi142.stdout:ID="rhel" 2024-04-10T22:39:00.121 INFO:teuthology.orchestra.run.smithi142.stdout:ID_LIKE="fedora" 2024-04-10T22:39:00.121 INFO:teuthology.orchestra.run.smithi142.stdout:VERSION_ID="8.6" 2024-04-10T22:39:00.121 INFO:teuthology.orchestra.run.smithi142.stdout:PLATFORM_ID="platform:el8" 2024-04-10T22:39:00.121 INFO:teuthology.orchestra.run.smithi142.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2024-04-10T22:39:00.121 INFO:teuthology.orchestra.run.smithi142.stdout:ANSI_COLOR="0;31" 2024-04-10T22:39:00.121 INFO:teuthology.orchestra.run.smithi142.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2024-04-10T22:39:00.121 INFO:teuthology.orchestra.run.smithi142.stdout:HOME_URL="https://www.redhat.com/" 2024-04-10T22:39:00.121 INFO:teuthology.orchestra.run.smithi142.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2024-04-10T22:39:00.121 INFO:teuthology.orchestra.run.smithi142.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-10T22:39:00.121 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-04-10T22:39:00.121 INFO:teuthology.orchestra.run.smithi142.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2024-04-10T22:39:00.122 INFO:teuthology.orchestra.run.smithi142.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2024-04-10T22:39:00.122 INFO:teuthology.orchestra.run.smithi142.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2024-04-10T22:39:00.122 INFO:teuthology.orchestra.run.smithi142.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2024-04-10T22:39:00.122 INFO:teuthology.lock.ops:Updating smithi142.front.sepia.ceph.com on lock server 2024-04-10T22:39:00.140 DEBUG:teuthology.orchestra.run.smithi150:> uname -m 2024-04-10T22:39:00.160 INFO:teuthology.orchestra.run.smithi150.stdout:x86_64 2024-04-10T22:39:00.161 DEBUG:teuthology.orchestra.run.smithi150:> cat /etc/os-release 2024-04-10T22:39:00.221 INFO:teuthology.orchestra.run.smithi150.stdout:NAME="Red Hat Enterprise Linux" 2024-04-10T22:39:00.221 INFO:teuthology.orchestra.run.smithi150.stdout:VERSION="8.6 (Ootpa)" 2024-04-10T22:39:00.221 INFO:teuthology.orchestra.run.smithi150.stdout:ID="rhel" 2024-04-10T22:39:00.221 INFO:teuthology.orchestra.run.smithi150.stdout:ID_LIKE="fedora" 2024-04-10T22:39:00.221 INFO:teuthology.orchestra.run.smithi150.stdout:VERSION_ID="8.6" 2024-04-10T22:39:00.221 INFO:teuthology.orchestra.run.smithi150.stdout:PLATFORM_ID="platform:el8" 2024-04-10T22:39:00.221 INFO:teuthology.orchestra.run.smithi150.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2024-04-10T22:39:00.221 INFO:teuthology.orchestra.run.smithi150.stdout:ANSI_COLOR="0;31" 2024-04-10T22:39:00.221 INFO:teuthology.orchestra.run.smithi150.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2024-04-10T22:39:00.222 INFO:teuthology.orchestra.run.smithi150.stdout:HOME_URL="https://www.redhat.com/" 2024-04-10T22:39:00.222 INFO:teuthology.orchestra.run.smithi150.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2024-04-10T22:39:00.222 INFO:teuthology.orchestra.run.smithi150.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-10T22:39:00.222 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-10T22:39:00.222 INFO:teuthology.orchestra.run.smithi150.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2024-04-10T22:39:00.222 INFO:teuthology.orchestra.run.smithi150.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2024-04-10T22:39:00.222 INFO:teuthology.orchestra.run.smithi150.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2024-04-10T22:39:00.222 INFO:teuthology.orchestra.run.smithi150.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2024-04-10T22:39:00.222 INFO:teuthology.lock.ops:Updating smithi150.front.sepia.ceph.com on lock server 2024-04-10T22:39:00.240 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-10T22:39:00.249 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-10T22:39:00.254 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-10T22:39:00.254 DEBUG:teuthology.orchestra.run.smithi142:> test '!' -e /home/ubuntu/cephtest 2024-04-10T22:39:00.257 DEBUG:teuthology.orchestra.run.smithi150:> test '!' -e /home/ubuntu/cephtest 2024-04-10T22:39:00.280 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-10T22:39:00.286 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-10T22:39:00.286 DEBUG:teuthology.orchestra.run.smithi142:> test -z $(ls -A /var/lib/ceph) 2024-04-10T22:39:00.315 DEBUG:teuthology.orchestra.run.smithi150:> test -z $(ls -A /var/lib/ceph) 2024-04-10T22:39:00.345 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-10T22:39:00.401 INFO:teuthology.run_tasks:Running task kernel... 2024-04-10T22:39:00.416 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-04-10T22:39:00.416 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-04-10T22:39:00.417 DEBUG:teuthology.orchestra.run.smithi142:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-10T22:39:00.417 DEBUG:teuthology.orchestra.run.smithi150:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-10T22:39:00.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:39:00.435 DEBUG:teuthology.orchestra.run.smithi150:> uname -r 2024-04-10T22:39:00.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:39:00.436 DEBUG:teuthology.orchestra.run.smithi142:> uname -r 2024-04-10T22:39:00.492 INFO:teuthology.orchestra.run.smithi150.stdout:4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:39:00.493 INFO:teuthology.task.kernel:Running kernel on smithi150: 4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:39:00.493 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum install -y kernel 2024-04-10T22:39:00.495 INFO:teuthology.orchestra.run.smithi142.stdout:4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:39:00.496 INFO:teuthology.task.kernel:Running kernel on smithi142: 4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:39:00.496 DEBUG:teuthology.orchestra.run.smithi142:> sudo yum install -y kernel 2024-04-10T22:39:02.048 INFO:teuthology.orchestra.run.smithi150.stdout:Updating Subscription Management repositories. 2024-04-10T22:39:02.048 INFO:teuthology.orchestra.run.smithi150.stdout:Unable to read consumer identity 2024-04-10T22:39:02.048 INFO:teuthology.orchestra.run.smithi142.stdout:Updating Subscription Management repositories. 2024-04-10T22:39:02.048 INFO:teuthology.orchestra.run.smithi142.stdout:Unable to read consumer identity 2024-04-10T22:39:02.147 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-04-10T22:39:02.150 INFO:teuthology.orchestra.run.smithi142.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:39:02.150 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-04-10T22:39:02.159 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-10T22:39:02.160 INFO:teuthology.orchestra.run.smithi150.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:39:02.160 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-10T22:39:45.414 INFO:teuthology.orchestra.run.smithi142.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:41 2024-04-10T22:39:45.414 INFO:teuthology.orchestra.run.smithi142.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:39:45.414 INFO:teuthology.orchestra.run.smithi142.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:39:45.414 INFO:teuthology.orchestra.run.smithi142.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:39:46.757 INFO:teuthology.orchestra.run.smithi142.stdout:Extra Packages for Enterprise Linux 12 MB/s | 16 MB 00:01 2024-04-10T22:39:50.396 INFO:teuthology.orchestra.run.smithi142.stdout:lab-extras 421 kB/s | 24 kB 00:00 2024-04-10T22:39:50.471 INFO:teuthology.orchestra.run.smithi142.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:39:51.141 INFO:teuthology.orchestra.run.smithi142.stdout:Package kernel-4.18.0-513.9.1.el8_9.x86_64 is already installed. 2024-04-10T22:39:51.141 INFO:teuthology.orchestra.run.smithi142.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2024-04-10T22:39:51.207 INFO:teuthology.orchestra.run.smithi142.stdout:Dependencies resolved. 2024-04-10T22:39:51.208 INFO:teuthology.orchestra.run.smithi142.stdout:Nothing to do. 2024-04-10T22:39:51.208 INFO:teuthology.orchestra.run.smithi142.stdout:Complete! 2024-04-10T22:39:51.358 DEBUG:teuthology.orchestra.run.smithi142:> echo no | sudo yum reinstall kernel || true 2024-04-10T22:39:51.695 INFO:teuthology.orchestra.run.smithi142.stdout:Updating Subscription Management repositories. 2024-04-10T22:39:51.695 INFO:teuthology.orchestra.run.smithi142.stdout:Unable to read consumer identity 2024-04-10T22:39:51.758 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-04-10T22:39:51.758 INFO:teuthology.orchestra.run.smithi142.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:39:51.758 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-04-10T22:39:56.902 INFO:teuthology.orchestra.run.smithi150.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:53 2024-04-10T22:39:56.902 INFO:teuthology.orchestra.run.smithi150.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:39:56.902 INFO:teuthology.orchestra.run.smithi150.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:39:56.902 INFO:teuthology.orchestra.run.smithi150.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:39:57.905 INFO:teuthology.orchestra.run.smithi150.stdout:Extra Packages for Enterprise Linux 16 MB/s | 16 MB 00:01 2024-04-10T22:40:01.571 INFO:teuthology.orchestra.run.smithi150.stdout:lab-extras 343 kB/s | 24 kB 00:00 2024-04-10T22:40:01.646 INFO:teuthology.orchestra.run.smithi150.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:40:02.222 INFO:teuthology.orchestra.run.smithi150.stdout:Package kernel-4.18.0-513.9.1.el8_9.x86_64 is already installed. 2024-04-10T22:40:02.223 INFO:teuthology.orchestra.run.smithi150.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2024-04-10T22:40:02.298 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-04-10T22:40:02.299 INFO:teuthology.orchestra.run.smithi150.stdout:Nothing to do. 2024-04-10T22:40:02.299 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-04-10T22:40:02.459 DEBUG:teuthology.orchestra.run.smithi150:> echo no | sudo yum reinstall kernel || true 2024-04-10T22:40:02.802 INFO:teuthology.orchestra.run.smithi150.stdout:Updating Subscription Management repositories. 2024-04-10T22:40:02.802 INFO:teuthology.orchestra.run.smithi150.stdout:Unable to read consumer identity 2024-04-10T22:40:02.858 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-10T22:40:02.858 INFO:teuthology.orchestra.run.smithi150.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:40:02.858 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-10T22:40:35.839 INFO:teuthology.orchestra.run.smithi142.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:43 2024-04-10T22:40:35.839 INFO:teuthology.orchestra.run.smithi142.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:40:35.840 INFO:teuthology.orchestra.run.smithi142.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:40:35.840 INFO:teuthology.orchestra.run.smithi142.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:40:35.995 INFO:teuthology.orchestra.run.smithi142.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:40:35.996 INFO:teuthology.orchestra.run.smithi142.stdout:Last metadata expiration check: 0:00:45 ago on Wed 10 Apr 2024 10:39:50 PM UTC. 2024-04-10T22:40:36.083 INFO:teuthology.orchestra.run.smithi142.stdout:Installed package kernel-4.18.0-513.9.1.el8_9.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available. 2024-04-10T22:40:36.083 INFO:teuthology.orchestra.run.smithi142.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2024-04-10T22:40:36.125 INFO:teuthology.orchestra.run.smithi142.stderr:Error: No packages marked for reinstall. 2024-04-10T22:40:36.173 DEBUG:teuthology.orchestra.run.smithi142:> sudo yum reinstall -y kernel || true 2024-04-10T22:40:36.509 INFO:teuthology.orchestra.run.smithi142.stdout:Updating Subscription Management repositories. 2024-04-10T22:40:36.509 INFO:teuthology.orchestra.run.smithi142.stdout:Unable to read consumer identity 2024-04-10T22:40:36.563 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-04-10T22:40:36.563 INFO:teuthology.orchestra.run.smithi142.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:40:36.563 INFO:teuthology.orchestra.run.smithi142.stdout: 2024-04-10T22:40:45.938 INFO:teuthology.orchestra.run.smithi150.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:42 2024-04-10T22:40:45.938 INFO:teuthology.orchestra.run.smithi150.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:40:45.938 INFO:teuthology.orchestra.run.smithi150.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:40:45.939 INFO:teuthology.orchestra.run.smithi150.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:40:46.101 INFO:teuthology.orchestra.run.smithi150.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:40:46.102 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:00:45 ago on Wed 10 Apr 2024 10:40:01 PM UTC. 2024-04-10T22:40:46.186 INFO:teuthology.orchestra.run.smithi150.stdout:Installed package kernel-4.18.0-513.9.1.el8_9.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available. 2024-04-10T22:40:46.195 INFO:teuthology.orchestra.run.smithi150.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2024-04-10T22:40:46.241 INFO:teuthology.orchestra.run.smithi150.stderr:Error: No packages marked for reinstall. 2024-04-10T22:40:46.289 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum reinstall -y kernel || true 2024-04-10T22:40:46.630 INFO:teuthology.orchestra.run.smithi150.stdout:Updating Subscription Management repositories. 2024-04-10T22:40:46.630 INFO:teuthology.orchestra.run.smithi150.stdout:Unable to read consumer identity 2024-04-10T22:40:46.687 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-10T22:40:46.687 INFO:teuthology.orchestra.run.smithi150.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:40:46.687 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-10T22:41:16.034 INFO:teuthology.orchestra.run.smithi142.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:39 2024-04-10T22:41:16.035 INFO:teuthology.orchestra.run.smithi142.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:41:16.035 INFO:teuthology.orchestra.run.smithi142.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:41:16.035 INFO:teuthology.orchestra.run.smithi142.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:41:16.182 INFO:teuthology.orchestra.run.smithi142.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:41:16.182 INFO:teuthology.orchestra.run.smithi142.stdout:Last metadata expiration check: 0:01:26 ago on Wed 10 Apr 2024 10:39:50 PM UTC. 2024-04-10T22:41:16.250 INFO:teuthology.orchestra.run.smithi142.stdout:Installed package kernel-4.18.0-513.9.1.el8_9.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available. 2024-04-10T22:41:16.250 INFO:teuthology.orchestra.run.smithi142.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2024-04-10T22:41:16.282 INFO:teuthology.orchestra.run.smithi142.stderr:Error: No packages marked for reinstall. 2024-04-10T22:41:16.328 DEBUG:teuthology.orchestra.run.smithi142:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:41:16.485 INFO:teuthology.orchestra.run.smithi142.stdout:kernel-4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:41:16.485 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:41:16.485 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-10T22:41:16.485 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-04-10T22:41:16.485 DEBUG:teuthology.orchestra.run.smithi142:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-10T22:41:16.545 INFO:teuthology.orchestra.run.smithi142.stdout:ttyS1 2024-04-10T22:41:16.556 DEBUG:teuthology.parallel:result is None 2024-04-10T22:41:26.152 INFO:teuthology.orchestra.run.smithi150.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:39 2024-04-10T22:41:26.153 INFO:teuthology.orchestra.run.smithi150.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:41:26.153 INFO:teuthology.orchestra.run.smithi150.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:41:26.153 INFO:teuthology.orchestra.run.smithi150.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:41:26.304 INFO:teuthology.orchestra.run.smithi150.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:41:26.304 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:01:25 ago on Wed 10 Apr 2024 10:40:01 PM UTC. 2024-04-10T22:41:26.366 INFO:teuthology.orchestra.run.smithi150.stdout:Installed package kernel-4.18.0-513.9.1.el8_9.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available. 2024-04-10T22:41:26.366 INFO:teuthology.orchestra.run.smithi150.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2024-04-10T22:41:26.407 INFO:teuthology.orchestra.run.smithi150.stderr:Error: No packages marked for reinstall. 2024-04-10T22:41:26.453 DEBUG:teuthology.orchestra.run.smithi150:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:41:26.605 INFO:teuthology.orchestra.run.smithi150.stdout:kernel-4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:41:26.605 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:41:26.605 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-10T22:41:26.605 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-04-10T22:41:26.605 DEBUG:teuthology.orchestra.run.smithi150:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-10T22:41:26.643 INFO:teuthology.orchestra.run.smithi150.stdout:ttyS1 2024-04-10T22:41:26.663 DEBUG:teuthology.parallel:result is None 2024-04-10T22:41:26.663 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-10T22:41:26.674 INFO:teuthology.task.internal:Creating test directory... 2024-04-10T22:41:26.674 DEBUG:teuthology.orchestra.run.smithi142:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-10T22:41:26.676 DEBUG:teuthology.orchestra.run.smithi150:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-10T22:41:26.719 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-10T22:41:26.730 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-10T22:41:26.743 INFO:teuthology.task.internal:Creating archive directory... 2024-04-10T22:41:26.744 DEBUG:teuthology.orchestra.run.smithi142:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-10T22:41:26.746 DEBUG:teuthology.orchestra.run.smithi150:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-10T22:41:26.791 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-10T22:41:26.799 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-10T22:41:26.800 DEBUG:teuthology.orchestra.run.smithi142:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-10T22:41:26.812 DEBUG:teuthology.orchestra.run.smithi150:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-10T22:41:26.850 INFO:teuthology.orchestra.run.smithi142.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:41:26.862 INFO:teuthology.orchestra.run.smithi142.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:41:26.863 INFO:teuthology.orchestra.run.smithi150.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:41:26.874 INFO:teuthology.orchestra.run.smithi150.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:41:26.876 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-10T22:41:26.884 INFO:teuthology.task.internal:Configuring sudo... 2024-04-10T22:41:26.884 DEBUG:teuthology.orchestra.run.smithi142:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-10T22:41:26.906 DEBUG:teuthology.orchestra.run.smithi150:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-10T22:41:26.942 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-10T22:41:26.953 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-10T22:41:26.953 DEBUG:teuthology.orchestra.run.smithi142:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-10T22:41:26.976 DEBUG:teuthology.orchestra.run.smithi150:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-10T22:41:26.999 DEBUG:teuthology.orchestra.run.smithi142:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:41:27.079 DEBUG:teuthology.orchestra.run.smithi142:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:41:27.136 DEBUG:teuthology.orchestra.run.smithi142:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:41:27.178 DEBUG:teuthology.orchestra.run.smithi142:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:41:27.223 DEBUG:teuthology.orchestra.run.smithi142:> set -ex 2024-04-10T22:41:27.223 DEBUG:teuthology.orchestra.run.smithi142:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-10T22:41:27.290 DEBUG:teuthology.orchestra.run.smithi150:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:41:27.335 DEBUG:teuthology.orchestra.run.smithi150:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:41:27.385 DEBUG:teuthology.orchestra.run.smithi150:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:41:27.471 DEBUG:teuthology.orchestra.run.smithi150:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:41:27.510 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-04-10T22:41:27.511 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-10T22:41:27.577 DEBUG:teuthology.orchestra.run.smithi142:> sudo service rsyslog restart 2024-04-10T22:41:27.580 DEBUG:teuthology.orchestra.run.smithi150:> sudo service rsyslog restart 2024-04-10T22:41:27.638 INFO:teuthology.orchestra.run.smithi142.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:41:27.789 INFO:teuthology.orchestra.run.smithi150.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:41:28.091 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-10T22:41:28.099 INFO:teuthology.task.internal:Starting timer... 2024-04-10T22:41:28.100 INFO:teuthology.run_tasks:Running task pcp... 2024-04-10T22:41:28.112 INFO:teuthology.run_tasks:Running task selinux... 2024-04-10T22:41:28.121 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-04-10T22:41:28.122 DEBUG:teuthology.orchestra.run.smithi142:> sudo service auditd rotate 2024-04-10T22:41:28.191 INFO:teuthology.orchestra.run.smithi142.stdout:Rotating logs: 2024-04-10T22:41:28.193 DEBUG:teuthology.orchestra.run.smithi150:> sudo service auditd rotate 2024-04-10T22:41:28.267 INFO:teuthology.orchestra.run.smithi150.stdout:Rotating logs: 2024-04-10T22:41:28.269 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-10T22:41:28.269 DEBUG:teuthology.orchestra.run.smithi142:> /usr/sbin/getenforce 2024-04-10T22:41:28.304 INFO:teuthology.orchestra.run.smithi142.stdout:Permissive 2024-04-10T22:41:28.305 DEBUG:teuthology.orchestra.run.smithi150:> /usr/sbin/getenforce 2024-04-10T22:41:28.327 INFO:teuthology.orchestra.run.smithi150.stdout:Permissive 2024-04-10T22:41:28.327 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi142.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi150.front.sepia.ceph.com': 'permissive'} 2024-04-10T22:41:28.328 DEBUG:teuthology.orchestra.run.smithi142:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:41:28.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:41:28.371 DEBUG:teuthology.orchestra.run.smithi150:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:41:28.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:41:28.400 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-10T22:41:28.400 DEBUG:teuthology.orchestra.run.smithi142:> sudo /usr/sbin/setenforce permissive 2024-04-10T22:41:28.439 DEBUG:teuthology.orchestra.run.smithi150:> sudo /usr/sbin/setenforce permissive 2024-04-10T22:41:28.471 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-10T22:41:28.481 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-04-10T22:41:28.486 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-04-10T22:41:28.609 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-10T22:41:28.627 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-10T22:41:28.628 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi142.front.sepia.ceph.com,smithi150.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-10T22:57:42.577 INFO:teuthology.task.ansible:Archiving ansible failure log at: /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650196/ansible_failures.yaml 2024-04-10T22:57:42.579 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/__init__.py", line 123, in __enter__ self.begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 508, in begin super(CephLab, self).begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 342, in begin self.execute_playbook() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 374, in execute_playbook self._handle_failure(command, status) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 402, in _handle_failure raise AnsibleFailedError(failures) teuthology.exceptions.AnsibleFailedError: [',', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at'] 2024-04-10T22:57:42.801 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=a7136fa80348454db7173286c24f7b90 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/__init__.py", line 123, in __enter__ self.begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 508, in begin super(CephLab, self).begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 342, in begin self.execute_playbook() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 374, in execute_playbook self._handle_failure(command, status) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 402, in _handle_failure raise AnsibleFailedError(failures) teuthology.exceptions.AnsibleFailedError: [',', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at'] 2024-04-10T22:57:42.803 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-10T22:57:42.812 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-10T22:57:42.812 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-10T22:57:42.820 DEBUG:teuthology.orchestra.run.smithi142:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-10T22:57:42.905 DEBUG:teuthology.orchestra.run.smithi150:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-10T22:57:42.971 DEBUG:teuthology.orchestra.run.smithi142:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:57:43.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:57:43.004 DEBUG:teuthology.orchestra.run.smithi150:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:57:43.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:57:43.034 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-10T22:57:43.043 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-10T22:57:43.051 INFO:teuthology.task.internal:Duration was 974.951479 seconds 2024-04-10T22:57:43.051 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-10T22:57:43.059 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-10T22:57:43.060 DEBUG:teuthology.orchestra.run.smithi142:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-10T22:57:43.062 DEBUG:teuthology.orchestra.run.smithi150:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-10T22:57:43.110 INFO:teuthology.orchestra.run.smithi142.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:57:43.122 INFO:teuthology.orchestra.run.smithi150.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:57:43.425 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-10T22:57:43.425 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi142.front.sepia.ceph.com 2024-04-10T22:57:43.425 DEBUG:teuthology.orchestra.run.smithi142:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-10T22:57:43.447 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi150.front.sepia.ceph.com 2024-04-10T22:57:43.448 DEBUG:teuthology.orchestra.run.smithi150:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-10T22:57:43.470 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-10T22:57:43.470 DEBUG:teuthology.orchestra.run.smithi142:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-10T22:57:43.490 DEBUG:teuthology.orchestra.run.smithi150:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-10T22:57:43.552 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-10T22:57:43.552 DEBUG:teuthology.orchestra.run.smithi142:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-10T22:57:43.573 DEBUG:teuthology.orchestra.run.smithi150:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-10T22:57:43.658 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-10T22:57:43.667 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-10T22:57:43.668 DEBUG:teuthology.orchestra.run.smithi142:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-10T22:57:43.694 DEBUG:teuthology.orchestra.run.smithi150:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-10T22:57:43.722 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-10T22:57:43.730 DEBUG:teuthology.orchestra.run.smithi142:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-10T22:57:43.737 DEBUG:teuthology.orchestra.run.smithi150:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-10T22:57:43.760 INFO:teuthology.orchestra.run.smithi142.stdout:kernel.core_pattern = core 2024-04-10T22:57:43.788 INFO:teuthology.orchestra.run.smithi150.stdout:kernel.core_pattern = core 2024-04-10T22:57:43.828 DEBUG:teuthology.orchestra.run.smithi142:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-10T22:57:43.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:57:43.848 DEBUG:teuthology.orchestra.run.smithi150:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-10T22:57:43.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:57:43.885 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-10T22:57:43.894 INFO:teuthology.task.internal:Transferring archived files... 2024-04-10T22:57:43.896 DEBUG:teuthology.misc:Transferring archived files from smithi142:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650196/remote/smithi142 2024-04-10T22:57:43.896 DEBUG:teuthology.orchestra.run.smithi142:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-10T22:57:43.940 DEBUG:teuthology.misc:Transferring archived files from smithi150:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650196/remote/smithi150 2024-04-10T22:57:43.941 DEBUG:teuthology.orchestra.run.smithi150:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-10T22:57:44.005 INFO:teuthology.task.internal:Removing archive directory... 2024-04-10T22:57:44.006 DEBUG:teuthology.orchestra.run.smithi142:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-10T22:57:44.008 DEBUG:teuthology.orchestra.run.smithi150:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-10T22:57:44.024 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-10T22:57:44.035 INFO:teuthology.task.internal:Not uploading archives. 2024-04-10T22:57:44.035 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-10T22:57:44.044 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-10T22:57:44.044 DEBUG:teuthology.orchestra.run.smithi142:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-10T22:57:44.065 DEBUG:teuthology.orchestra.run.smithi150:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-10T22:57:44.079 INFO:teuthology.orchestra.run.smithi142.stdout: 262171 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 10 22:57 /home/ubuntu/cephtest 2024-04-10T22:57:44.082 INFO:teuthology.orchestra.run.smithi150.stdout: 262171 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 10 22:57 /home/ubuntu/cephtest 2024-04-10T22:57:44.083 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-10T22:57:44.092 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-10T22:57:44.136 INFO:teuthology.nuke:Checking targets against current locks 2024-04-10T22:57:44.168 DEBUG:teuthology.nuke:shortname: smithi142 2024-04-10T22:57:44.168 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T22:57:44.210 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi142.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650196', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:22.121620', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBH9asL/GBiwXWLhxPgHH+NR8jEaZdBitJfqCxhsRrgZ+v0tOjf++dShNsuyZGcze5lLUJldJuyMM/AlMugoW01M='} 2024-04-10T22:57:44.225 DEBUG:teuthology.nuke:shortname: smithi150 2024-04-10T22:57:44.225 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T22:57:44.254 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi150.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650196', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:22.123101', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNSRi03CHd0+TAOz3vlIezFRx1PaZsdNziFiTMZ6Da27IzlTvod6VGT8wjlDWPOCEtQNKMvd/TRY67kZeNQq1Pk='} 2024-04-10T22:57:44.308 INFO:teuthology.orchestra.console.smithi142:Power off 2024-04-10T22:57:44.308 DEBUG:teuthology.orchestra.console.smithi142:pexpect command: ipmitool -H smithi142.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-10T22:57:44.357 INFO:teuthology.orchestra.console.smithi150:Power off 2024-04-10T22:57:44.358 DEBUG:teuthology.orchestra.console.smithi150:pexpect command: ipmitool -H smithi150.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-10T22:57:44.368 DEBUG:teuthology.orchestra.console.smithi142:power off output: Chassis Power Control: Down/Off 2024-04-10T22:57:44.369 DEBUG:teuthology.orchestra.console.smithi142:pexpect command: ipmitool -H smithi142.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:57:44.412 DEBUG:teuthology.orchestra.console.smithi150:power off output: Chassis Power Control: Down/Off 2024-04-10T22:57:44.412 DEBUG:teuthology.orchestra.console.smithi150:pexpect command: ipmitool -H smithi150.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:57:44.423 DEBUG:teuthology.orchestra.console.smithi142:check power output: Chassis Power is on 2024-04-10T22:57:44.468 DEBUG:teuthology.orchestra.console.smithi150:check power output: Chassis Power is on 2024-04-10T22:57:48.424 DEBUG:teuthology.orchestra.console.smithi142:pexpect command: ipmitool -H smithi142.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:57:48.469 DEBUG:teuthology.orchestra.console.smithi150:pexpect command: ipmitool -H smithi150.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:57:48.535 DEBUG:teuthology.orchestra.console.smithi142:check power output: Chassis Power is on 2024-04-10T22:57:48.579 DEBUG:teuthology.orchestra.console.smithi150:check power output: Chassis Power is on 2024-04-10T22:57:52.538 DEBUG:teuthology.orchestra.console.smithi142:pexpect command: ipmitool -H smithi142.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:57:52.581 DEBUG:teuthology.orchestra.console.smithi150:pexpect command: ipmitool -H smithi150.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:57:52.649 DEBUG:teuthology.orchestra.console.smithi142:check power output: Chassis Power is on 2024-04-10T22:57:52.692 DEBUG:teuthology.orchestra.console.smithi150:check power output: Chassis Power is on 2024-04-10T22:57:56.655 DEBUG:teuthology.orchestra.console.smithi142:pexpect command: ipmitool -H smithi142.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:57:56.692 DEBUG:teuthology.orchestra.console.smithi150:pexpect command: ipmitool -H smithi150.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:57:56.764 DEBUG:teuthology.orchestra.console.smithi142:check power output: Chassis Power is off 2024-04-10T22:57:56.803 DEBUG:teuthology.orchestra.console.smithi150:check power output: Chassis Power is off 2024-04-10T22:57:56.866 INFO:teuthology.orchestra.console.smithi142:Power off completed 2024-04-10T22:57:56.904 INFO:teuthology.orchestra.console.smithi150:Power off completed 2024-04-10T22:57:57.010 INFO:teuthology.run:Summary data: description: upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_rhel8 0-roles 1-start 2-first-half-tasks/readwrite 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} duration: 974.9514789581299 failure_reason: '['','', ''Bugzilla. In case of problems, contact the owner of this repository.'', ''Enabling a Copr repository. Please note that this repository is not part'', ''Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable'', ''Please do not file bug reports about these packages in Fedora'', ''The Fedora Project does not exercise any power over the contents of'', ''and packages are not held to any quality or security level.'', ''of the main distribution, and quality may vary.'', ''this repository beyond the rules outlined in the Copr FAQ at'']' owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=a7136fa80348454db7173286c24f7b90 status: dead success: false 2024-04-10T22:57:57.010 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-10T22:57:57.104 INFO:teuthology.run:DEAD