2024-04-10T22:22:59.771 INFO:root:teuthology version: 0.0.1.dev267+g6c63784 2024-04-10T22:22:59.772 DEBUG:teuthology.run:Teuthology command: teuthology --verbose --archive /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650170 --description upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} --owner scheduled_teuthology@teuthology --name teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi -- /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650170/orig.config.yaml 2024-04-10T22:22:59.828 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-10T22:22:59.923 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650170 branch: reef description: upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} email: ceph-qa@ceph.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' first_in_suite: false job_id: '7650170' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run basic import/export cli tests for rbd ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: rhel os_version: '8.6' overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: reef sha1: 4f70a9a90881107996c8b7a3e89ba8b32624713c owner: scheduled_teuthology@teuthology priority: 830 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 4604 sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: pacific clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 0/1 suite: upgrade:pacific-x suite_branch: reef suite_path: /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 4f70a9a90881107996c8b7a3e89ba8b32624713c targets: smithi107.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPVwkv2F5TQ2LjQP1Y0csr6lDFSVHKuFU3FzFMxVfYXqPHUnj6JCnDZvDfn0ewxn4xNVOMeukDn53GwiIYgaBO4= smithi148.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBC42qG1FnWfVaWRT5g5jTU8Kc826lOTXYS3NDZeW0ALsI/bEfRP2oY3L0uNTNXo8HX/K2z3Xw5cjlm0cri30kP0= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools:3.0 --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - install: branch: pacific exclude_packages: - ceph-volume - cephadm: cephadm_branch: pacific cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:pacific - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 6c637841c215537a4502385240412f1966e0faab timestamp: 2024-04-10_01:08:08 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-04-10T22:22:59.923 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa; will attempt to use it 2024-04-10T22:22:59.924 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa/tasks 2024-04-10T22:22:59.924 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-10T22:22:59.925 INFO:teuthology.task.internal:Checking packages... 2024-04-10T22:22:59.948 INFO:teuthology.task.internal:Checking packages for os_type 'rhel', flavor 'default' and ceph hash 'd540ebaca6b131a1dd560e7f69e024b133bbaa42' 2024-04-10T22:22:59.948 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-10T22:22:59.948 INFO:teuthology.packaging:ref: None 2024-04-10T22:22:59.948 INFO:teuthology.packaging:tag: None 2024-04-10T22:22:59.948 INFO:teuthology.packaging:branch: reef 2024-04-10T22:22:59.948 INFO:teuthology.packaging:sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 2024-04-10T22:22:59.948 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef 2024-04-10T22:23:00.440 INFO:teuthology.task.internal:Found packages for ceph version 18.2.2-1189.gc96c8d47 2024-04-10T22:23:00.441 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-10T22:23:00.448 INFO:teuthology.task.internal:no buildpackages task found 2024-04-10T22:23:00.448 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-10T22:23:00.466 INFO:teuthology.task.internal:Saving configuration 2024-04-10T22:23:00.486 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-10T22:23:00.494 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T22:23:00.519 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi107.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650170', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:15:49.304644', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPVwkv2F5TQ2LjQP1Y0csr6lDFSVHKuFU3FzFMxVfYXqPHUnj6JCnDZvDfn0ewxn4xNVOMeukDn53GwiIYgaBO4='} 2024-04-10T22:23:00.543 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi148.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650170', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:15:49.302274', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBC42qG1FnWfVaWRT5g5jTU8Kc826lOTXYS3NDZeW0ALsI/bEfRP2oY3L0uNTNXo8HX/K2z3Xw5cjlm0cri30kP0='} 2024-04-10T22:23:00.543 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-10T22:23:00.549 INFO:teuthology.task.internal:roles: ubuntu@smithi107.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-04-10T22:23:00.549 INFO:teuthology.task.internal:roles: ubuntu@smithi148.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-04-10T22:23:00.549 INFO:teuthology.run_tasks:Running task console_log... 2024-04-10T22:23:00.614 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f385a34c0d0>, signals=[15]) 2024-04-10T22:23:00.614 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-10T22:23:00.620 INFO:teuthology.task.internal:Opening connections... 2024-04-10T22:23:00.620 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi107.front.sepia.ceph.com 2024-04-10T22:23:00.622 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi107.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:23:00.696 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi148.front.sepia.ceph.com 2024-04-10T22:23:00.697 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi148.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:23:00.768 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-10T22:23:00.775 DEBUG:teuthology.orchestra.run.smithi107:> uname -m 2024-04-10T22:23:00.797 INFO:teuthology.orchestra.run.smithi107.stdout:x86_64 2024-04-10T22:23:00.797 DEBUG:teuthology.orchestra.run.smithi107:> cat /etc/os-release 2024-04-10T22:23:00.857 INFO:teuthology.orchestra.run.smithi107.stdout:NAME="Red Hat Enterprise Linux" 2024-04-10T22:23:00.857 INFO:teuthology.orchestra.run.smithi107.stdout:VERSION="8.6 (Ootpa)" 2024-04-10T22:23:00.857 INFO:teuthology.orchestra.run.smithi107.stdout:ID="rhel" 2024-04-10T22:23:00.857 INFO:teuthology.orchestra.run.smithi107.stdout:ID_LIKE="fedora" 2024-04-10T22:23:00.857 INFO:teuthology.orchestra.run.smithi107.stdout:VERSION_ID="8.6" 2024-04-10T22:23:00.857 INFO:teuthology.orchestra.run.smithi107.stdout:PLATFORM_ID="platform:el8" 2024-04-10T22:23:00.857 INFO:teuthology.orchestra.run.smithi107.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2024-04-10T22:23:00.858 INFO:teuthology.orchestra.run.smithi107.stdout:ANSI_COLOR="0;31" 2024-04-10T22:23:00.858 INFO:teuthology.orchestra.run.smithi107.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2024-04-10T22:23:00.858 INFO:teuthology.orchestra.run.smithi107.stdout:HOME_URL="https://www.redhat.com/" 2024-04-10T22:23:00.858 INFO:teuthology.orchestra.run.smithi107.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2024-04-10T22:23:00.858 INFO:teuthology.orchestra.run.smithi107.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-10T22:23:00.858 INFO:teuthology.orchestra.run.smithi107.stdout: 2024-04-10T22:23:00.858 INFO:teuthology.orchestra.run.smithi107.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2024-04-10T22:23:00.858 INFO:teuthology.orchestra.run.smithi107.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2024-04-10T22:23:00.858 INFO:teuthology.orchestra.run.smithi107.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2024-04-10T22:23:00.858 INFO:teuthology.orchestra.run.smithi107.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2024-04-10T22:23:00.859 INFO:teuthology.lock.ops:Updating smithi107.front.sepia.ceph.com on lock server 2024-04-10T22:23:00.882 DEBUG:teuthology.orchestra.run.smithi148:> uname -m 2024-04-10T22:23:00.902 INFO:teuthology.orchestra.run.smithi148.stdout:x86_64 2024-04-10T22:23:00.903 DEBUG:teuthology.orchestra.run.smithi148:> cat /etc/os-release 2024-04-10T22:23:00.962 INFO:teuthology.orchestra.run.smithi148.stdout:NAME="Red Hat Enterprise Linux" 2024-04-10T22:23:00.963 INFO:teuthology.orchestra.run.smithi148.stdout:VERSION="8.6 (Ootpa)" 2024-04-10T22:23:00.963 INFO:teuthology.orchestra.run.smithi148.stdout:ID="rhel" 2024-04-10T22:23:00.963 INFO:teuthology.orchestra.run.smithi148.stdout:ID_LIKE="fedora" 2024-04-10T22:23:00.963 INFO:teuthology.orchestra.run.smithi148.stdout:VERSION_ID="8.6" 2024-04-10T22:23:00.963 INFO:teuthology.orchestra.run.smithi148.stdout:PLATFORM_ID="platform:el8" 2024-04-10T22:23:00.963 INFO:teuthology.orchestra.run.smithi148.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2024-04-10T22:23:00.963 INFO:teuthology.orchestra.run.smithi148.stdout:ANSI_COLOR="0;31" 2024-04-10T22:23:00.963 INFO:teuthology.orchestra.run.smithi148.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2024-04-10T22:23:00.963 INFO:teuthology.orchestra.run.smithi148.stdout:HOME_URL="https://www.redhat.com/" 2024-04-10T22:23:00.963 INFO:teuthology.orchestra.run.smithi148.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2024-04-10T22:23:00.963 INFO:teuthology.orchestra.run.smithi148.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-10T22:23:00.964 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-04-10T22:23:00.964 INFO:teuthology.orchestra.run.smithi148.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2024-04-10T22:23:00.964 INFO:teuthology.orchestra.run.smithi148.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2024-04-10T22:23:00.964 INFO:teuthology.orchestra.run.smithi148.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2024-04-10T22:23:00.964 INFO:teuthology.orchestra.run.smithi148.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2024-04-10T22:23:00.964 INFO:teuthology.lock.ops:Updating smithi148.front.sepia.ceph.com on lock server 2024-04-10T22:23:01.107 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-10T22:23:01.116 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-10T22:23:01.121 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-10T22:23:01.122 DEBUG:teuthology.orchestra.run.smithi107:> test '!' -e /home/ubuntu/cephtest 2024-04-10T22:23:01.124 DEBUG:teuthology.orchestra.run.smithi148:> test '!' -e /home/ubuntu/cephtest 2024-04-10T22:23:01.143 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-10T22:23:01.149 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-10T22:23:01.149 DEBUG:teuthology.orchestra.run.smithi107:> test -z $(ls -A /var/lib/ceph) 2024-04-10T22:23:01.183 DEBUG:teuthology.orchestra.run.smithi148:> test -z $(ls -A /var/lib/ceph) 2024-04-10T22:23:01.214 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-10T22:23:01.266 INFO:teuthology.run_tasks:Running task kernel... 2024-04-10T22:23:01.280 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-04-10T22:23:01.280 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-04-10T22:23:01.280 DEBUG:teuthology.orchestra.run.smithi107:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-10T22:23:01.281 DEBUG:teuthology.orchestra.run.smithi148:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-10T22:23:01.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:23:01.299 DEBUG:teuthology.orchestra.run.smithi107:> uname -r 2024-04-10T22:23:01.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:23:01.300 DEBUG:teuthology.orchestra.run.smithi148:> uname -r 2024-04-10T22:23:01.358 INFO:teuthology.orchestra.run.smithi107.stdout:4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:23:01.359 INFO:teuthology.task.kernel:Running kernel on smithi107: 4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:23:01.359 DEBUG:teuthology.orchestra.run.smithi107:> sudo yum install -y kernel 2024-04-10T22:23:01.360 INFO:teuthology.orchestra.run.smithi148.stdout:4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:23:01.360 INFO:teuthology.task.kernel:Running kernel on smithi148: 4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:23:01.360 DEBUG:teuthology.orchestra.run.smithi148:> sudo yum install -y kernel 2024-04-10T22:23:02.873 INFO:teuthology.orchestra.run.smithi107.stdout:Updating Subscription Management repositories. 2024-04-10T22:23:02.873 INFO:teuthology.orchestra.run.smithi107.stdout:Unable to read consumer identity 2024-04-10T22:23:02.890 INFO:teuthology.orchestra.run.smithi148.stdout:Updating Subscription Management repositories. 2024-04-10T22:23:02.890 INFO:teuthology.orchestra.run.smithi148.stdout:Unable to read consumer identity 2024-04-10T22:23:02.997 INFO:teuthology.orchestra.run.smithi107.stdout: 2024-04-10T22:23:02.998 INFO:teuthology.orchestra.run.smithi107.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:23:02.998 INFO:teuthology.orchestra.run.smithi107.stdout: 2024-04-10T22:23:03.020 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-04-10T22:23:03.020 INFO:teuthology.orchestra.run.smithi148.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:23:03.020 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-04-10T22:23:42.000 INFO:teuthology.orchestra.run.smithi148.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:37 2024-04-10T22:23:42.000 INFO:teuthology.orchestra.run.smithi148.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:23:42.000 INFO:teuthology.orchestra.run.smithi148.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:23:42.000 INFO:teuthology.orchestra.run.smithi148.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:23:42.027 INFO:teuthology.orchestra.run.smithi107.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:37 2024-04-10T22:23:42.027 INFO:teuthology.orchestra.run.smithi107.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:23:42.027 INFO:teuthology.orchestra.run.smithi107.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:23:42.028 INFO:teuthology.orchestra.run.smithi107.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:25:53.153 INFO:teuthology.orchestra.run.smithi107.stdout:Extra Packages for Enterprise Linux 127 kB/s | 16 MB 02:11 2024-04-10T22:25:53.381 INFO:teuthology.orchestra.run.smithi148.stdout:Extra Packages for Enterprise Linux 127 kB/s | 16 MB 02:11 2024-04-10T22:25:56.794 INFO:teuthology.orchestra.run.smithi107.stdout:lab-extras 244 kB/s | 24 kB 00:00 2024-04-10T22:25:56.869 INFO:teuthology.orchestra.run.smithi107.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:25:57.062 INFO:teuthology.orchestra.run.smithi148.stdout:lab-extras 548 kB/s | 24 kB 00:00 2024-04-10T22:25:57.137 INFO:teuthology.orchestra.run.smithi148.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:25:57.488 INFO:teuthology.orchestra.run.smithi107.stdout:Package kernel-4.18.0-513.9.1.el8_9.x86_64 is already installed. 2024-04-10T22:25:57.488 INFO:teuthology.orchestra.run.smithi107.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2024-04-10T22:25:57.564 INFO:teuthology.orchestra.run.smithi107.stdout:Dependencies resolved. 2024-04-10T22:25:57.564 INFO:teuthology.orchestra.run.smithi107.stdout:Nothing to do. 2024-04-10T22:25:57.565 INFO:teuthology.orchestra.run.smithi107.stdout:Complete! 2024-04-10T22:25:57.693 DEBUG:teuthology.orchestra.run.smithi107:> echo no | sudo yum reinstall kernel || true 2024-04-10T22:25:57.765 INFO:teuthology.orchestra.run.smithi148.stdout:Package kernel-4.18.0-513.9.1.el8_9.x86_64 is already installed. 2024-04-10T22:25:57.765 INFO:teuthology.orchestra.run.smithi148.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2024-04-10T22:25:57.839 INFO:teuthology.orchestra.run.smithi148.stdout:Dependencies resolved. 2024-04-10T22:25:57.840 INFO:teuthology.orchestra.run.smithi148.stdout:Nothing to do. 2024-04-10T22:25:57.840 INFO:teuthology.orchestra.run.smithi148.stdout:Complete! 2024-04-10T22:25:57.969 DEBUG:teuthology.orchestra.run.smithi148:> echo no | sudo yum reinstall kernel || true 2024-04-10T22:25:58.029 INFO:teuthology.orchestra.run.smithi107.stdout:Updating Subscription Management repositories. 2024-04-10T22:25:58.029 INFO:teuthology.orchestra.run.smithi107.stdout:Unable to read consumer identity 2024-04-10T22:25:58.089 INFO:teuthology.orchestra.run.smithi107.stdout: 2024-04-10T22:25:58.089 INFO:teuthology.orchestra.run.smithi107.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:25:58.089 INFO:teuthology.orchestra.run.smithi107.stdout: 2024-04-10T22:25:58.309 INFO:teuthology.orchestra.run.smithi148.stdout:Updating Subscription Management repositories. 2024-04-10T22:25:58.309 INFO:teuthology.orchestra.run.smithi148.stdout:Unable to read consumer identity 2024-04-10T22:25:58.364 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-04-10T22:25:58.365 INFO:teuthology.orchestra.run.smithi148.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:25:58.365 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-04-10T22:26:36.560 INFO:teuthology.orchestra.run.smithi107.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:38 2024-04-10T22:26:36.561 INFO:teuthology.orchestra.run.smithi107.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:26:36.561 INFO:teuthology.orchestra.run.smithi107.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:26:36.561 INFO:teuthology.orchestra.run.smithi107.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:26:36.636 INFO:teuthology.orchestra.run.smithi148.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:38 2024-04-10T22:26:36.636 INFO:teuthology.orchestra.run.smithi148.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:26:36.636 INFO:teuthology.orchestra.run.smithi148.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:26:36.636 INFO:teuthology.orchestra.run.smithi148.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:26:36.710 INFO:teuthology.orchestra.run.smithi107.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:26:36.710 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:00:40 ago on Wed 10 Apr 2024 10:25:56 PM UTC. 2024-04-10T22:26:36.786 INFO:teuthology.orchestra.run.smithi107.stdout:Installed package kernel-4.18.0-513.9.1.el8_9.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available. 2024-04-10T22:26:36.788 INFO:teuthology.orchestra.run.smithi107.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2024-04-10T22:26:36.793 INFO:teuthology.orchestra.run.smithi148.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:26:36.793 INFO:teuthology.orchestra.run.smithi148.stdout:Last metadata expiration check: 0:00:39 ago on Wed 10 Apr 2024 10:25:57 PM UTC. 2024-04-10T22:26:36.832 INFO:teuthology.orchestra.run.smithi107.stderr:Error: No packages marked for reinstall. 2024-04-10T22:26:36.878 DEBUG:teuthology.orchestra.run.smithi107:> sudo yum reinstall -y kernel || true 2024-04-10T22:26:36.891 INFO:teuthology.orchestra.run.smithi148.stdout:Installed package kernel-4.18.0-513.9.1.el8_9.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available. 2024-04-10T22:26:36.891 INFO:teuthology.orchestra.run.smithi148.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2024-04-10T22:26:36.926 INFO:teuthology.orchestra.run.smithi148.stderr:Error: No packages marked for reinstall. 2024-04-10T22:26:36.975 DEBUG:teuthology.orchestra.run.smithi148:> sudo yum reinstall -y kernel || true 2024-04-10T22:26:37.215 INFO:teuthology.orchestra.run.smithi107.stdout:Updating Subscription Management repositories. 2024-04-10T22:26:37.215 INFO:teuthology.orchestra.run.smithi107.stdout:Unable to read consumer identity 2024-04-10T22:26:37.276 INFO:teuthology.orchestra.run.smithi107.stdout: 2024-04-10T22:26:37.277 INFO:teuthology.orchestra.run.smithi107.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:26:37.277 INFO:teuthology.orchestra.run.smithi107.stdout: 2024-04-10T22:26:37.312 INFO:teuthology.orchestra.run.smithi148.stdout:Updating Subscription Management repositories. 2024-04-10T22:26:37.312 INFO:teuthology.orchestra.run.smithi148.stdout:Unable to read consumer identity 2024-04-10T22:26:37.368 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-04-10T22:26:37.368 INFO:teuthology.orchestra.run.smithi148.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:26:37.368 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-04-10T22:27:17.734 INFO:teuthology.orchestra.run.smithi107.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:40 2024-04-10T22:27:17.735 INFO:teuthology.orchestra.run.smithi107.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:27:17.735 INFO:teuthology.orchestra.run.smithi107.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:27:17.735 INFO:teuthology.orchestra.run.smithi107.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:27:17.840 INFO:teuthology.orchestra.run.smithi148.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:40 2024-04-10T22:27:17.841 INFO:teuthology.orchestra.run.smithi148.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:27:17.841 INFO:teuthology.orchestra.run.smithi148.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:27:17.841 INFO:teuthology.orchestra.run.smithi148.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:27:17.881 INFO:teuthology.orchestra.run.smithi107.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:27:17.882 INFO:teuthology.orchestra.run.smithi107.stdout:Last metadata expiration check: 0:01:21 ago on Wed 10 Apr 2024 10:25:56 PM UTC. 2024-04-10T22:27:17.945 INFO:teuthology.orchestra.run.smithi107.stdout:Installed package kernel-4.18.0-513.9.1.el8_9.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available. 2024-04-10T22:27:17.945 INFO:teuthology.orchestra.run.smithi107.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2024-04-10T22:27:17.979 INFO:teuthology.orchestra.run.smithi107.stderr:Error: No packages marked for reinstall. 2024-04-10T22:27:17.992 INFO:teuthology.orchestra.run.smithi148.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:27:17.992 INFO:teuthology.orchestra.run.smithi148.stdout:Last metadata expiration check: 0:01:20 ago on Wed 10 Apr 2024 10:25:57 PM UTC. 2024-04-10T22:27:18.025 DEBUG:teuthology.orchestra.run.smithi107:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:27:18.065 INFO:teuthology.orchestra.run.smithi148.stdout:Installed package kernel-4.18.0-513.9.1.el8_9.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available. 2024-04-10T22:27:18.065 INFO:teuthology.orchestra.run.smithi148.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2024-04-10T22:27:18.093 INFO:teuthology.orchestra.run.smithi148.stderr:Error: No packages marked for reinstall. 2024-04-10T22:27:18.140 DEBUG:teuthology.orchestra.run.smithi148:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:27:18.194 INFO:teuthology.orchestra.run.smithi107.stdout:kernel-4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:27:18.195 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:27:18.195 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-10T22:27:18.195 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-04-10T22:27:18.195 DEBUG:teuthology.orchestra.run.smithi107:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-10T22:27:18.245 INFO:teuthology.orchestra.run.smithi107.stdout:ttyS1 2024-04-10T22:27:18.259 DEBUG:teuthology.parallel:result is None 2024-04-10T22:27:18.293 INFO:teuthology.orchestra.run.smithi148.stdout:kernel-4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:27:18.293 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:27:18.294 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-10T22:27:18.294 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-04-10T22:27:18.294 DEBUG:teuthology.orchestra.run.smithi148:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-10T22:27:18.330 INFO:teuthology.orchestra.run.smithi148.stdout:ttyS1 2024-04-10T22:27:18.342 DEBUG:teuthology.parallel:result is None 2024-04-10T22:27:18.343 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-10T22:27:18.353 INFO:teuthology.task.internal:Creating test directory... 2024-04-10T22:27:18.353 DEBUG:teuthology.orchestra.run.smithi107:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-10T22:27:18.356 DEBUG:teuthology.orchestra.run.smithi148:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-10T22:27:18.400 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-10T22:27:18.407 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-10T22:27:18.446 INFO:teuthology.task.internal:Creating archive directory... 2024-04-10T22:27:18.446 DEBUG:teuthology.orchestra.run.smithi107:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-10T22:27:18.448 DEBUG:teuthology.orchestra.run.smithi148:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-10T22:27:18.487 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-10T22:27:18.494 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-10T22:27:18.495 DEBUG:teuthology.orchestra.run.smithi107:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-10T22:27:18.515 DEBUG:teuthology.orchestra.run.smithi148:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-10T22:27:18.559 INFO:teuthology.orchestra.run.smithi107.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:27:18.560 INFO:teuthology.orchestra.run.smithi148.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:27:18.564 INFO:teuthology.orchestra.run.smithi107.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:27:18.570 INFO:teuthology.orchestra.run.smithi148.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:27:18.572 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-10T22:27:18.579 INFO:teuthology.task.internal:Configuring sudo... 2024-04-10T22:27:18.580 DEBUG:teuthology.orchestra.run.smithi107:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-10T22:27:18.608 DEBUG:teuthology.orchestra.run.smithi148:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-10T22:27:18.639 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-10T22:27:18.649 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-10T22:27:18.649 DEBUG:teuthology.orchestra.run.smithi107:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-10T22:27:18.677 DEBUG:teuthology.orchestra.run.smithi148:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-10T22:27:18.696 DEBUG:teuthology.orchestra.run.smithi107:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:27:18.779 DEBUG:teuthology.orchestra.run.smithi107:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:27:18.831 DEBUG:teuthology.orchestra.run.smithi107:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:27:18.915 DEBUG:teuthology.orchestra.run.smithi107:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:27:18.957 DEBUG:teuthology.orchestra.run.smithi107:> set -ex 2024-04-10T22:27:18.957 DEBUG:teuthology.orchestra.run.smithi107:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-10T22:27:19.024 DEBUG:teuthology.orchestra.run.smithi148:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:27:19.070 DEBUG:teuthology.orchestra.run.smithi148:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:27:19.122 DEBUG:teuthology.orchestra.run.smithi148:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:27:19.206 DEBUG:teuthology.orchestra.run.smithi148:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:27:19.249 DEBUG:teuthology.orchestra.run.smithi148:> set -ex 2024-04-10T22:27:19.249 DEBUG:teuthology.orchestra.run.smithi148:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-10T22:27:19.315 DEBUG:teuthology.orchestra.run.smithi107:> sudo service rsyslog restart 2024-04-10T22:27:19.318 DEBUG:teuthology.orchestra.run.smithi148:> sudo service rsyslog restart 2024-04-10T22:27:19.363 INFO:teuthology.orchestra.run.smithi107.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:27:19.477 INFO:teuthology.orchestra.run.smithi148.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:27:19.788 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-10T22:27:19.796 INFO:teuthology.task.internal:Starting timer... 2024-04-10T22:27:19.796 INFO:teuthology.run_tasks:Running task pcp... 2024-04-10T22:27:19.865 INFO:teuthology.run_tasks:Running task selinux... 2024-04-10T22:27:19.874 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-04-10T22:27:19.874 DEBUG:teuthology.orchestra.run.smithi107:> sudo service auditd rotate 2024-04-10T22:27:19.944 INFO:teuthology.orchestra.run.smithi107.stdout:Rotating logs: 2024-04-10T22:27:19.947 DEBUG:teuthology.orchestra.run.smithi148:> sudo service auditd rotate 2024-04-10T22:27:20.001 INFO:teuthology.orchestra.run.smithi148.stdout:Rotating logs: 2024-04-10T22:27:20.003 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-10T22:27:20.003 DEBUG:teuthology.orchestra.run.smithi107:> /usr/sbin/getenforce 2024-04-10T22:27:20.032 INFO:teuthology.orchestra.run.smithi107.stdout:Permissive 2024-04-10T22:27:20.032 DEBUG:teuthology.orchestra.run.smithi148:> /usr/sbin/getenforce 2024-04-10T22:27:20.058 INFO:teuthology.orchestra.run.smithi148.stdout:Permissive 2024-04-10T22:27:20.058 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi107.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi148.front.sepia.ceph.com': 'permissive'} 2024-04-10T22:27:20.059 DEBUG:teuthology.orchestra.run.smithi107:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:27:20.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:27:20.099 DEBUG:teuthology.orchestra.run.smithi148:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:27:20.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:27:20.127 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-10T22:27:20.127 DEBUG:teuthology.orchestra.run.smithi107:> sudo /usr/sbin/setenforce permissive 2024-04-10T22:27:20.186 DEBUG:teuthology.orchestra.run.smithi148:> sudo /usr/sbin/setenforce permissive 2024-04-10T22:27:20.219 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-10T22:27:20.230 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-10T22:27:20.246 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-10T22:27:20.247 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi107.front.sepia.ceph.com,smithi148.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-10T22:43:08.574 INFO:teuthology.task.ansible:Archiving ansible failure log at: /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650170/ansible_failures.yaml 2024-04-10T22:43:08.576 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/__init__.py", line 123, in __enter__ self.begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 508, in begin super(CephLab, self).begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 342, in begin self.execute_playbook() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 374, in execute_playbook self._handle_failure(command, status) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 402, in _handle_failure raise AnsibleFailedError(failures) teuthology.exceptions.AnsibleFailedError: [',', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at'] 2024-04-10T22:43:08.821 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=95c41539844f404f9ff0e5fbbbd28dcc Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/__init__.py", line 123, in __enter__ self.begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 508, in begin super(CephLab, self).begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 342, in begin self.execute_playbook() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 374, in execute_playbook self._handle_failure(command, status) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 402, in _handle_failure raise AnsibleFailedError(failures) teuthology.exceptions.AnsibleFailedError: [',', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at'] 2024-04-10T22:43:08.823 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-10T22:43:08.831 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-10T22:43:08.832 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-10T22:43:08.852 DEBUG:teuthology.orchestra.run.smithi107:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-10T22:43:09.005 DEBUG:teuthology.orchestra.run.smithi148:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-10T22:43:09.072 DEBUG:teuthology.orchestra.run.smithi107:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:43:09.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:43:09.104 DEBUG:teuthology.orchestra.run.smithi148:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:43:09.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:43:09.135 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-10T22:43:09.145 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-10T22:43:09.158 INFO:teuthology.task.internal:Duration was 949.362322 seconds 2024-04-10T22:43:09.159 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-10T22:43:09.173 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-10T22:43:09.173 DEBUG:teuthology.orchestra.run.smithi107:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-10T22:43:09.176 DEBUG:teuthology.orchestra.run.smithi148:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-10T22:43:09.223 INFO:teuthology.orchestra.run.smithi107.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:43:09.224 INFO:teuthology.orchestra.run.smithi148.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:43:09.503 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-10T22:43:09.503 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi107.front.sepia.ceph.com 2024-04-10T22:43:09.503 DEBUG:teuthology.orchestra.run.smithi107:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-10T22:43:09.530 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi148.front.sepia.ceph.com 2024-04-10T22:43:09.530 DEBUG:teuthology.orchestra.run.smithi148:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-10T22:43:09.552 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-10T22:43:09.553 DEBUG:teuthology.orchestra.run.smithi107:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-10T22:43:09.573 DEBUG:teuthology.orchestra.run.smithi148:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-10T22:43:09.634 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-10T22:43:09.634 DEBUG:teuthology.orchestra.run.smithi107:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-10T22:43:09.638 DEBUG:teuthology.orchestra.run.smithi148:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-10T22:43:09.754 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-10T22:43:09.763 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-10T22:43:09.763 DEBUG:teuthology.orchestra.run.smithi107:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-10T22:43:09.790 DEBUG:teuthology.orchestra.run.smithi148:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-10T22:43:09.817 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-10T22:43:09.827 DEBUG:teuthology.orchestra.run.smithi107:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-10T22:43:09.832 DEBUG:teuthology.orchestra.run.smithi148:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-10T22:43:09.856 INFO:teuthology.orchestra.run.smithi107.stdout:kernel.core_pattern = core 2024-04-10T22:43:09.882 INFO:teuthology.orchestra.run.smithi148.stdout:kernel.core_pattern = core 2024-04-10T22:43:09.909 DEBUG:teuthology.orchestra.run.smithi107:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-10T22:43:09.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:43:09.937 DEBUG:teuthology.orchestra.run.smithi148:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-10T22:43:09.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:43:09.965 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-10T22:43:09.975 INFO:teuthology.task.internal:Transferring archived files... 2024-04-10T22:43:09.976 DEBUG:teuthology.misc:Transferring archived files from smithi107:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650170/remote/smithi107 2024-04-10T22:43:09.977 DEBUG:teuthology.orchestra.run.smithi107:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-10T22:43:10.026 DEBUG:teuthology.misc:Transferring archived files from smithi148:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650170/remote/smithi148 2024-04-10T22:43:10.027 DEBUG:teuthology.orchestra.run.smithi148:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-10T22:43:10.082 INFO:teuthology.task.internal:Removing archive directory... 2024-04-10T22:43:10.082 DEBUG:teuthology.orchestra.run.smithi107:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-10T22:43:10.085 DEBUG:teuthology.orchestra.run.smithi148:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-10T22:43:10.101 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-10T22:43:10.112 INFO:teuthology.task.internal:Not uploading archives. 2024-04-10T22:43:10.113 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-10T22:43:10.124 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-10T22:43:10.125 DEBUG:teuthology.orchestra.run.smithi107:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-10T22:43:10.142 DEBUG:teuthology.orchestra.run.smithi148:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-10T22:43:10.156 INFO:teuthology.orchestra.run.smithi107.stdout: 262171 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 10 22:43 /home/ubuntu/cephtest 2024-04-10T22:43:10.158 INFO:teuthology.orchestra.run.smithi148.stdout: 262171 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 10 22:43 /home/ubuntu/cephtest 2024-04-10T22:43:10.159 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-10T22:43:10.170 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-10T22:43:10.195 INFO:teuthology.nuke:Checking targets against current locks 2024-04-10T22:43:10.239 DEBUG:teuthology.nuke:shortname: smithi107 2024-04-10T22:43:10.240 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T22:43:10.346 DEBUG:teuthology.nuke:shortname: smithi148 2024-04-10T22:43:10.346 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T22:43:10.386 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi107.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650170', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:15:49.304644', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPVwkv2F5TQ2LjQP1Y0csr6lDFSVHKuFU3FzFMxVfYXqPHUnj6JCnDZvDfn0ewxn4xNVOMeukDn53GwiIYgaBO4='} 2024-04-10T22:43:10.499 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi148.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650170', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:15:49.302274', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBC42qG1FnWfVaWRT5g5jTU8Kc826lOTXYS3NDZeW0ALsI/bEfRP2oY3L0uNTNXo8HX/K2z3Xw5cjlm0cri30kP0='} 2024-04-10T22:43:10.569 INFO:teuthology.orchestra.console.smithi107:Power off 2024-04-10T22:43:10.569 DEBUG:teuthology.orchestra.console.smithi107:pexpect command: ipmitool -H smithi107.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-10T22:43:10.648 INFO:teuthology.orchestra.console.smithi148:Power off 2024-04-10T22:43:10.649 DEBUG:teuthology.orchestra.console.smithi148:pexpect command: ipmitool -H smithi148.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-10T22:43:10.658 DEBUG:teuthology.orchestra.console.smithi107:power off output: Chassis Power Control: Down/Off 2024-04-10T22:43:10.659 DEBUG:teuthology.orchestra.console.smithi107:pexpect command: ipmitool -H smithi107.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:43:10.725 DEBUG:teuthology.orchestra.console.smithi107:check power output: Chassis Power is on 2024-04-10T22:43:10.737 DEBUG:teuthology.orchestra.console.smithi148:power off output: Chassis Power Control: Down/Off 2024-04-10T22:43:10.737 DEBUG:teuthology.orchestra.console.smithi148:pexpect command: ipmitool -H smithi148.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:43:10.820 DEBUG:teuthology.orchestra.console.smithi148:check power output: Chassis Power is on 2024-04-10T22:43:14.727 DEBUG:teuthology.orchestra.console.smithi107:pexpect command: ipmitool -H smithi107.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:43:14.821 DEBUG:teuthology.orchestra.console.smithi148:pexpect command: ipmitool -H smithi148.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:43:14.838 DEBUG:teuthology.orchestra.console.smithi107:check power output: Chassis Power is on 2024-04-10T22:43:14.931 DEBUG:teuthology.orchestra.console.smithi148:check power output: Chassis Power is on 2024-04-10T22:43:18.842 DEBUG:teuthology.orchestra.console.smithi107:pexpect command: ipmitool -H smithi107.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:43:18.932 DEBUG:teuthology.orchestra.console.smithi148:pexpect command: ipmitool -H smithi148.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:43:18.953 DEBUG:teuthology.orchestra.console.smithi107:check power output: Chassis Power is off 2024-04-10T22:43:19.043 DEBUG:teuthology.orchestra.console.smithi148:check power output: Chassis Power is off 2024-04-10T22:43:19.057 INFO:teuthology.orchestra.console.smithi107:Power off completed 2024-04-10T22:43:19.144 INFO:teuthology.orchestra.console.smithi148:Power off completed 2024-04-10T22:43:19.251 INFO:teuthology.run:Summary data: description: upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} duration: 949.3623223304749 failure_reason: '['','', ''Bugzilla. In case of problems, contact the owner of this repository.'', ''Enabling a Copr repository. Please note that this repository is not part'', ''Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable'', ''Please do not file bug reports about these packages in Fedora'', ''The Fedora Project does not exercise any power over the contents of'', ''and packages are not held to any quality or security level.'', ''of the main distribution, and quality may vary.'', ''this repository beyond the rules outlined in the Copr FAQ at'']' owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=95c41539844f404f9ff0e5fbbbd28dcc status: dead success: false 2024-04-10T22:43:19.251 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-10T22:43:19.355 INFO:teuthology.run:DEAD