2024-04-10T22:37:54.631 INFO:root:teuthology version: 0.0.1.dev267+g6c63784 2024-04-10T22:37:54.632 DEBUG:teuthology.run:Teuthology command: teuthology --archive /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650195 --name teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi --verbose --description upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} --owner scheduled_teuthology@teuthology -- /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650195/orig.config.yaml 2024-04-10T22:37:54.686 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-10T22:37:54.806 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650195 branch: reef description: upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} email: ceph-qa@ceph.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: pacific clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' first_in_suite: false job_id: '7650195' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'librbd C and C++ api tests ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run basic import/export cli tests for rbd ' name: teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: rhel os_version: '8.6' overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: reef sha1: 4f70a9a90881107996c8b7a3e89ba8b32624713c owner: scheduled_teuthology@teuthology priority: 830 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' seed: 4604 sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: pacific clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 0/1 suite: upgrade:pacific-x suite_branch: reef suite_path: /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 4f70a9a90881107996c8b7a3e89ba8b32624713c targets: smithi055.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMPHzA4Wo1NnldvhC/y5ufPFUpgEiYnwZHiRSfM1A/wg91wGYf05SsIf56GtgYs/p+zUWOeBBOAfxn5K+816ams= smithi116.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFKILFoaEe+TTWRXVQWo8PI7J5YbBh/JQ+BHUCwIlG3GVpuyVw0DMZCCIV1g8Ehdx+KLXiGhEryPYLYPgA9YN7U= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools:3.0 --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - install: branch: pacific exclude_packages: - ceph-volume - cephadm: cephadm_branch: pacific cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:pacific - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 6c637841c215537a4502385240412f1966e0faab timestamp: 2024-04-10_01:08:08 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-04-10T22:37:54.806 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa; will attempt to use it 2024-04-10T22:37:54.806 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa/tasks 2024-04-10T22:37:54.807 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-10T22:37:54.808 INFO:teuthology.task.internal:Checking packages... 2024-04-10T22:37:54.832 INFO:teuthology.task.internal:Checking packages for os_type 'rhel', flavor 'default' and ceph hash 'd540ebaca6b131a1dd560e7f69e024b133bbaa42' 2024-04-10T22:37:54.832 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-10T22:37:54.832 INFO:teuthology.packaging:ref: None 2024-04-10T22:37:54.832 INFO:teuthology.packaging:tag: None 2024-04-10T22:37:54.832 INFO:teuthology.packaging:branch: reef 2024-04-10T22:37:54.832 INFO:teuthology.packaging:sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 2024-04-10T22:37:54.832 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef 2024-04-10T22:37:55.152 INFO:teuthology.task.internal:Found packages for ceph version 18.2.2-1189.gc96c8d47 2024-04-10T22:37:55.153 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-10T22:37:55.159 INFO:teuthology.task.internal:no buildpackages task found 2024-04-10T22:37:55.160 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-10T22:37:55.276 INFO:teuthology.task.internal:Saving configuration 2024-04-10T22:37:55.297 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-10T22:37:55.303 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T22:37:55.328 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi055.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650195', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:21.720259', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMPHzA4Wo1NnldvhC/y5ufPFUpgEiYnwZHiRSfM1A/wg91wGYf05SsIf56GtgYs/p+zUWOeBBOAfxn5K+816ams='} 2024-04-10T22:37:55.349 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi116.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650195', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:21.721542', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFKILFoaEe+TTWRXVQWo8PI7J5YbBh/JQ+BHUCwIlG3GVpuyVw0DMZCCIV1g8Ehdx+KLXiGhEryPYLYPgA9YN7U='} 2024-04-10T22:37:55.349 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-10T22:37:55.356 INFO:teuthology.task.internal:roles: ubuntu@smithi055.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-04-10T22:37:55.356 INFO:teuthology.task.internal:roles: ubuntu@smithi116.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-04-10T22:37:55.356 INFO:teuthology.run_tasks:Running task console_log... 2024-04-10T22:37:55.427 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f1b199ac0d0>, signals=[15]) 2024-04-10T22:37:55.427 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-10T22:37:55.434 INFO:teuthology.task.internal:Opening connections... 2024-04-10T22:37:55.435 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi055.front.sepia.ceph.com 2024-04-10T22:37:55.436 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi055.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:37:55.511 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi116.front.sepia.ceph.com 2024-04-10T22:37:55.512 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi116.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:37:55.586 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-10T22:37:55.593 DEBUG:teuthology.orchestra.run.smithi055:> uname -m 2024-04-10T22:37:55.616 INFO:teuthology.orchestra.run.smithi055.stdout:x86_64 2024-04-10T22:37:55.616 DEBUG:teuthology.orchestra.run.smithi055:> cat /etc/os-release 2024-04-10T22:37:55.678 INFO:teuthology.orchestra.run.smithi055.stdout:NAME="Red Hat Enterprise Linux" 2024-04-10T22:37:55.678 INFO:teuthology.orchestra.run.smithi055.stdout:VERSION="8.6 (Ootpa)" 2024-04-10T22:37:55.678 INFO:teuthology.orchestra.run.smithi055.stdout:ID="rhel" 2024-04-10T22:37:55.678 INFO:teuthology.orchestra.run.smithi055.stdout:ID_LIKE="fedora" 2024-04-10T22:37:55.678 INFO:teuthology.orchestra.run.smithi055.stdout:VERSION_ID="8.6" 2024-04-10T22:37:55.678 INFO:teuthology.orchestra.run.smithi055.stdout:PLATFORM_ID="platform:el8" 2024-04-10T22:37:55.678 INFO:teuthology.orchestra.run.smithi055.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2024-04-10T22:37:55.678 INFO:teuthology.orchestra.run.smithi055.stdout:ANSI_COLOR="0;31" 2024-04-10T22:37:55.679 INFO:teuthology.orchestra.run.smithi055.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2024-04-10T22:37:55.679 INFO:teuthology.orchestra.run.smithi055.stdout:HOME_URL="https://www.redhat.com/" 2024-04-10T22:37:55.679 INFO:teuthology.orchestra.run.smithi055.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2024-04-10T22:37:55.679 INFO:teuthology.orchestra.run.smithi055.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-10T22:37:55.679 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-10T22:37:55.679 INFO:teuthology.orchestra.run.smithi055.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2024-04-10T22:37:55.679 INFO:teuthology.orchestra.run.smithi055.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2024-04-10T22:37:55.679 INFO:teuthology.orchestra.run.smithi055.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2024-04-10T22:37:55.679 INFO:teuthology.orchestra.run.smithi055.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2024-04-10T22:37:55.680 INFO:teuthology.lock.ops:Updating smithi055.front.sepia.ceph.com on lock server 2024-04-10T22:37:55.699 DEBUG:teuthology.orchestra.run.smithi116:> uname -m 2024-04-10T22:37:55.719 INFO:teuthology.orchestra.run.smithi116.stdout:x86_64 2024-04-10T22:37:55.719 DEBUG:teuthology.orchestra.run.smithi116:> cat /etc/os-release 2024-04-10T22:37:55.777 INFO:teuthology.orchestra.run.smithi116.stdout:NAME="Red Hat Enterprise Linux" 2024-04-10T22:37:55.778 INFO:teuthology.orchestra.run.smithi116.stdout:VERSION="8.6 (Ootpa)" 2024-04-10T22:37:55.778 INFO:teuthology.orchestra.run.smithi116.stdout:ID="rhel" 2024-04-10T22:37:55.778 INFO:teuthology.orchestra.run.smithi116.stdout:ID_LIKE="fedora" 2024-04-10T22:37:55.778 INFO:teuthology.orchestra.run.smithi116.stdout:VERSION_ID="8.6" 2024-04-10T22:37:55.778 INFO:teuthology.orchestra.run.smithi116.stdout:PLATFORM_ID="platform:el8" 2024-04-10T22:37:55.778 INFO:teuthology.orchestra.run.smithi116.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2024-04-10T22:37:55.778 INFO:teuthology.orchestra.run.smithi116.stdout:ANSI_COLOR="0;31" 2024-04-10T22:37:55.778 INFO:teuthology.orchestra.run.smithi116.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2024-04-10T22:37:55.778 INFO:teuthology.orchestra.run.smithi116.stdout:HOME_URL="https://www.redhat.com/" 2024-04-10T22:37:55.778 INFO:teuthology.orchestra.run.smithi116.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2024-04-10T22:37:55.778 INFO:teuthology.orchestra.run.smithi116.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-10T22:37:55.778 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-10T22:37:55.779 INFO:teuthology.orchestra.run.smithi116.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2024-04-10T22:37:55.779 INFO:teuthology.orchestra.run.smithi116.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2024-04-10T22:37:55.779 INFO:teuthology.orchestra.run.smithi116.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2024-04-10T22:37:55.779 INFO:teuthology.orchestra.run.smithi116.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2024-04-10T22:37:55.779 INFO:teuthology.lock.ops:Updating smithi116.front.sepia.ceph.com on lock server 2024-04-10T22:37:55.798 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-10T22:37:55.807 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-10T22:37:55.813 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-10T22:37:55.814 DEBUG:teuthology.orchestra.run.smithi055:> test '!' -e /home/ubuntu/cephtest 2024-04-10T22:37:55.816 DEBUG:teuthology.orchestra.run.smithi116:> test '!' -e /home/ubuntu/cephtest 2024-04-10T22:37:55.835 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-10T22:37:55.841 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-10T22:37:55.841 DEBUG:teuthology.orchestra.run.smithi055:> test -z $(ls -A /var/lib/ceph) 2024-04-10T22:37:55.877 DEBUG:teuthology.orchestra.run.smithi116:> test -z $(ls -A /var/lib/ceph) 2024-04-10T22:37:55.904 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-10T22:37:55.958 INFO:teuthology.run_tasks:Running task kernel... 2024-04-10T22:37:55.974 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-04-10T22:37:55.974 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-04-10T22:37:55.974 DEBUG:teuthology.orchestra.run.smithi055:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-10T22:37:55.975 DEBUG:teuthology.orchestra.run.smithi116:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-10T22:37:55.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:37:55.992 DEBUG:teuthology.orchestra.run.smithi116:> uname -r 2024-04-10T22:37:55.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:37:55.995 DEBUG:teuthology.orchestra.run.smithi055:> uname -r 2024-04-10T22:37:56.050 INFO:teuthology.orchestra.run.smithi116.stdout:4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:37:56.050 INFO:teuthology.task.kernel:Running kernel on smithi116: 4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:37:56.050 DEBUG:teuthology.orchestra.run.smithi116:> sudo yum install -y kernel 2024-04-10T22:37:56.055 INFO:teuthology.orchestra.run.smithi055.stdout:4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:37:56.055 INFO:teuthology.task.kernel:Running kernel on smithi055: 4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:37:56.055 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum install -y kernel 2024-04-10T22:37:57.541 INFO:teuthology.orchestra.run.smithi116.stdout:Updating Subscription Management repositories. 2024-04-10T22:37:57.541 INFO:teuthology.orchestra.run.smithi116.stdout:Unable to read consumer identity 2024-04-10T22:37:57.614 INFO:teuthology.orchestra.run.smithi055.stdout:Updating Subscription Management repositories. 2024-04-10T22:37:57.615 INFO:teuthology.orchestra.run.smithi055.stdout:Unable to read consumer identity 2024-04-10T22:37:57.662 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-10T22:37:57.662 INFO:teuthology.orchestra.run.smithi116.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:37:57.662 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-10T22:37:57.744 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-10T22:37:57.744 INFO:teuthology.orchestra.run.smithi055.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:37:57.745 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-10T22:38:44.915 INFO:teuthology.orchestra.run.smithi116.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:46 2024-04-10T22:38:44.916 INFO:teuthology.orchestra.run.smithi116.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:38:44.916 INFO:teuthology.orchestra.run.smithi116.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:38:44.917 INFO:teuthology.orchestra.run.smithi116.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:38:45.051 INFO:teuthology.orchestra.run.smithi055.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:45 2024-04-10T22:38:45.051 INFO:teuthology.orchestra.run.smithi055.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:38:45.051 INFO:teuthology.orchestra.run.smithi055.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:38:45.052 INFO:teuthology.orchestra.run.smithi055.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:40:58.956 INFO:teuthology.orchestra.run.smithi116.stdout:Extra Packages for Enterprise Linux 124 kB/s | 16 MB 02:14 2024-04-10T22:41:02.531 INFO:teuthology.orchestra.run.smithi116.stdout:lab-extras 421 kB/s | 24 kB 00:00 2024-04-10T22:41:02.605 INFO:teuthology.orchestra.run.smithi116.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:41:03.233 INFO:teuthology.orchestra.run.smithi116.stdout:Package kernel-4.18.0-513.9.1.el8_9.x86_64 is already installed. 2024-04-10T22:41:03.234 INFO:teuthology.orchestra.run.smithi116.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2024-04-10T22:41:03.301 INFO:teuthology.orchestra.run.smithi116.stdout:Dependencies resolved. 2024-04-10T22:41:03.302 INFO:teuthology.orchestra.run.smithi116.stdout:Nothing to do. 2024-04-10T22:41:03.302 INFO:teuthology.orchestra.run.smithi116.stdout:Complete! 2024-04-10T22:41:03.424 DEBUG:teuthology.orchestra.run.smithi116:> echo no | sudo yum reinstall kernel || true 2024-04-10T22:41:03.801 INFO:teuthology.orchestra.run.smithi116.stdout:Updating Subscription Management repositories. 2024-04-10T22:41:03.801 INFO:teuthology.orchestra.run.smithi116.stdout:Unable to read consumer identity 2024-04-10T22:41:03.862 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-10T22:41:03.862 INFO:teuthology.orchestra.run.smithi116.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:41:03.862 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-10T22:41:46.339 INFO:teuthology.orchestra.run.smithi116.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:42 2024-04-10T22:41:46.340 INFO:teuthology.orchestra.run.smithi116.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:41:46.340 INFO:teuthology.orchestra.run.smithi116.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:41:46.340 INFO:teuthology.orchestra.run.smithi116.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:41:46.489 INFO:teuthology.orchestra.run.smithi116.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:41:46.489 INFO:teuthology.orchestra.run.smithi116.stdout:Last metadata expiration check: 0:00:44 ago on Wed 10 Apr 2024 10:41:02 PM UTC. 2024-04-10T22:41:46.562 INFO:teuthology.orchestra.run.smithi116.stdout:Installed package kernel-4.18.0-513.9.1.el8_9.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available. 2024-04-10T22:41:46.566 INFO:teuthology.orchestra.run.smithi116.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2024-04-10T22:41:46.606 INFO:teuthology.orchestra.run.smithi116.stderr:Error: No packages marked for reinstall. 2024-04-10T22:41:46.652 DEBUG:teuthology.orchestra.run.smithi116:> sudo yum reinstall -y kernel || true 2024-04-10T22:41:46.989 INFO:teuthology.orchestra.run.smithi116.stdout:Updating Subscription Management repositories. 2024-04-10T22:41:46.989 INFO:teuthology.orchestra.run.smithi116.stdout:Unable to read consumer identity 2024-04-10T22:41:47.049 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-10T22:41:47.049 INFO:teuthology.orchestra.run.smithi116.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:41:47.049 INFO:teuthology.orchestra.run.smithi116.stdout: 2024-04-10T22:42:23.073 INFO:teuthology.orchestra.run.smithi055.stdout:Extra Packages for Enterprise Linux 76 kB/s | 16 MB 03:38 2024-04-10T22:42:27.115 INFO:teuthology.orchestra.run.smithi055.stdout:lab-extras 409 kB/s | 24 kB 00:00 2024-04-10T22:42:27.126 INFO:teuthology.orchestra.run.smithi116.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:39 2024-04-10T22:42:27.126 INFO:teuthology.orchestra.run.smithi116.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:42:27.126 INFO:teuthology.orchestra.run.smithi116.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:42:27.126 INFO:teuthology.orchestra.run.smithi116.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:42:27.198 INFO:teuthology.orchestra.run.smithi055.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:42:27.272 INFO:teuthology.orchestra.run.smithi116.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:42:27.272 INFO:teuthology.orchestra.run.smithi116.stdout:Last metadata expiration check: 0:01:25 ago on Wed 10 Apr 2024 10:41:02 PM UTC. 2024-04-10T22:42:27.335 INFO:teuthology.orchestra.run.smithi116.stdout:Installed package kernel-4.18.0-513.9.1.el8_9.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available. 2024-04-10T22:42:27.335 INFO:teuthology.orchestra.run.smithi116.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2024-04-10T22:42:27.371 INFO:teuthology.orchestra.run.smithi116.stderr:Error: No packages marked for reinstall. 2024-04-10T22:42:27.418 DEBUG:teuthology.orchestra.run.smithi116:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:42:27.569 INFO:teuthology.orchestra.run.smithi116.stdout:kernel-4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:42:27.569 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:42:27.569 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-10T22:42:27.569 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-04-10T22:42:27.570 DEBUG:teuthology.orchestra.run.smithi116:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-10T22:42:27.606 INFO:teuthology.orchestra.run.smithi116.stdout:ttyS1 2024-04-10T22:42:27.620 DEBUG:teuthology.parallel:result is None 2024-04-10T22:42:27.901 INFO:teuthology.orchestra.run.smithi055.stdout:Package kernel-4.18.0-513.9.1.el8_9.x86_64 is already installed. 2024-04-10T22:42:27.901 INFO:teuthology.orchestra.run.smithi055.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2024-04-10T22:42:27.979 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2024-04-10T22:42:27.981 INFO:teuthology.orchestra.run.smithi055.stdout:Nothing to do. 2024-04-10T22:42:27.981 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2024-04-10T22:42:28.144 DEBUG:teuthology.orchestra.run.smithi055:> echo no | sudo yum reinstall kernel || true 2024-04-10T22:42:28.550 INFO:teuthology.orchestra.run.smithi055.stdout:Updating Subscription Management repositories. 2024-04-10T22:42:28.550 INFO:teuthology.orchestra.run.smithi055.stdout:Unable to read consumer identity 2024-04-10T22:42:28.611 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-10T22:42:28.611 INFO:teuthology.orchestra.run.smithi055.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:42:28.611 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-10T22:43:07.522 INFO:teuthology.orchestra.run.smithi055.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:38 2024-04-10T22:43:07.522 INFO:teuthology.orchestra.run.smithi055.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:43:07.523 INFO:teuthology.orchestra.run.smithi055.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:43:07.523 INFO:teuthology.orchestra.run.smithi055.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:43:07.694 INFO:teuthology.orchestra.run.smithi055.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:43:07.695 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:00:40 ago on Wed 10 Apr 2024 10:42:27 PM UTC. 2024-04-10T22:43:07.803 INFO:teuthology.orchestra.run.smithi055.stdout:Installed package kernel-4.18.0-513.9.1.el8_9.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available. 2024-04-10T22:43:07.810 INFO:teuthology.orchestra.run.smithi055.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2024-04-10T22:43:07.856 INFO:teuthology.orchestra.run.smithi055.stderr:Error: No packages marked for reinstall. 2024-04-10T22:43:07.911 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum reinstall -y kernel || true 2024-04-10T22:43:08.294 INFO:teuthology.orchestra.run.smithi055.stdout:Updating Subscription Management repositories. 2024-04-10T22:43:08.294 INFO:teuthology.orchestra.run.smithi055.stdout:Unable to read consumer identity 2024-04-10T22:43:08.348 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-10T22:43:08.349 INFO:teuthology.orchestra.run.smithi055.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:43:08.349 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-10T22:43:47.854 INFO:teuthology.orchestra.run.smithi055.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:39 2024-04-10T22:43:47.855 INFO:teuthology.orchestra.run.smithi055.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:43:47.855 INFO:teuthology.orchestra.run.smithi055.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:43:47.855 INFO:teuthology.orchestra.run.smithi055.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:43:48.013 INFO:teuthology.orchestra.run.smithi055.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:43:48.013 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:01:21 ago on Wed 10 Apr 2024 10:42:27 PM UTC. 2024-04-10T22:43:48.081 INFO:teuthology.orchestra.run.smithi055.stdout:Installed package kernel-4.18.0-513.9.1.el8_9.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available. 2024-04-10T22:43:48.082 INFO:teuthology.orchestra.run.smithi055.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2024-04-10T22:43:48.124 INFO:teuthology.orchestra.run.smithi055.stderr:Error: No packages marked for reinstall. 2024-04-10T22:43:48.182 DEBUG:teuthology.orchestra.run.smithi055:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:43:48.366 INFO:teuthology.orchestra.run.smithi055.stdout:kernel-4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:43:48.366 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:43:48.367 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-10T22:43:48.367 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-04-10T22:43:48.367 DEBUG:teuthology.orchestra.run.smithi055:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-10T22:43:48.407 INFO:teuthology.orchestra.run.smithi055.stdout:ttyS1 2024-04-10T22:43:48.427 DEBUG:teuthology.parallel:result is None 2024-04-10T22:43:48.427 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-10T22:43:48.436 INFO:teuthology.task.internal:Creating test directory... 2024-04-10T22:43:48.437 DEBUG:teuthology.orchestra.run.smithi055:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-10T22:43:48.469 DEBUG:teuthology.orchestra.run.smithi116:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-10T22:43:48.490 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-10T22:43:48.500 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-10T22:43:48.514 INFO:teuthology.task.internal:Creating archive directory... 2024-04-10T22:43:48.514 DEBUG:teuthology.orchestra.run.smithi055:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-10T22:43:48.530 DEBUG:teuthology.orchestra.run.smithi116:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-10T22:43:48.572 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-10T22:43:48.583 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-10T22:43:48.583 DEBUG:teuthology.orchestra.run.smithi055:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-10T22:43:48.609 DEBUG:teuthology.orchestra.run.smithi116:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-10T22:43:48.647 INFO:teuthology.orchestra.run.smithi055.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:43:48.656 INFO:teuthology.orchestra.run.smithi116.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:43:48.660 INFO:teuthology.orchestra.run.smithi055.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:43:48.668 INFO:teuthology.orchestra.run.smithi116.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:43:48.670 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-10T22:43:48.677 INFO:teuthology.task.internal:Configuring sudo... 2024-04-10T22:43:48.677 DEBUG:teuthology.orchestra.run.smithi055:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-10T22:43:48.705 DEBUG:teuthology.orchestra.run.smithi116:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-10T22:43:48.737 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-10T22:43:48.748 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-10T22:43:48.748 DEBUG:teuthology.orchestra.run.smithi055:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-10T22:43:48.781 DEBUG:teuthology.orchestra.run.smithi116:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-10T22:43:48.799 DEBUG:teuthology.orchestra.run.smithi055:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:43:48.904 DEBUG:teuthology.orchestra.run.smithi055:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:43:48.949 DEBUG:teuthology.orchestra.run.smithi055:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:43:49.054 DEBUG:teuthology.orchestra.run.smithi055:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:43:49.089 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2024-04-10T22:43:49.089 DEBUG:teuthology.orchestra.run.smithi055:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-10T22:43:49.163 DEBUG:teuthology.orchestra.run.smithi116:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:43:49.222 DEBUG:teuthology.orchestra.run.smithi116:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:43:49.269 DEBUG:teuthology.orchestra.run.smithi116:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:43:49.366 DEBUG:teuthology.orchestra.run.smithi116:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:43:49.398 DEBUG:teuthology.orchestra.run.smithi116:> set -ex 2024-04-10T22:43:49.398 DEBUG:teuthology.orchestra.run.smithi116:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-10T22:43:49.466 DEBUG:teuthology.orchestra.run.smithi055:> sudo service rsyslog restart 2024-04-10T22:43:49.469 DEBUG:teuthology.orchestra.run.smithi116:> sudo service rsyslog restart 2024-04-10T22:43:49.544 INFO:teuthology.orchestra.run.smithi055.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:43:49.615 INFO:teuthology.orchestra.run.smithi116.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:43:49.976 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-10T22:43:49.988 INFO:teuthology.task.internal:Starting timer... 2024-04-10T22:43:49.988 INFO:teuthology.run_tasks:Running task pcp... 2024-04-10T22:43:50.007 INFO:teuthology.run_tasks:Running task selinux... 2024-04-10T22:43:50.024 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-04-10T22:43:50.024 DEBUG:teuthology.orchestra.run.smithi055:> sudo service auditd rotate 2024-04-10T22:43:50.103 INFO:teuthology.orchestra.run.smithi055.stdout:Rotating logs: 2024-04-10T22:43:50.105 DEBUG:teuthology.orchestra.run.smithi116:> sudo service auditd rotate 2024-04-10T22:43:50.162 INFO:teuthology.orchestra.run.smithi116.stdout:Rotating logs: 2024-04-10T22:43:50.164 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-10T22:43:50.165 DEBUG:teuthology.orchestra.run.smithi055:> /usr/sbin/getenforce 2024-04-10T22:43:50.194 INFO:teuthology.orchestra.run.smithi055.stdout:Permissive 2024-04-10T22:43:50.194 DEBUG:teuthology.orchestra.run.smithi116:> /usr/sbin/getenforce 2024-04-10T22:43:50.218 INFO:teuthology.orchestra.run.smithi116.stdout:Permissive 2024-04-10T22:43:50.218 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi055.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi116.front.sepia.ceph.com': 'permissive'} 2024-04-10T22:43:50.219 DEBUG:teuthology.orchestra.run.smithi055:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:43:50.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:43:50.265 DEBUG:teuthology.orchestra.run.smithi116:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:43:50.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:43:50.293 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-10T22:43:50.293 DEBUG:teuthology.orchestra.run.smithi055:> sudo /usr/sbin/setenforce permissive 2024-04-10T22:43:50.337 DEBUG:teuthology.orchestra.run.smithi116:> sudo /usr/sbin/setenforce permissive 2024-04-10T22:43:50.364 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-10T22:43:50.375 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-04-10T22:43:50.380 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-04-10T22:43:50.569 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-10T22:43:50.585 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-10T22:43:50.587 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi055.front.sepia.ceph.com,smithi116.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-10T23:01:11.943 INFO:teuthology.task.ansible:Archiving ansible failure log at: /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650195/ansible_failures.yaml 2024-04-10T23:01:11.945 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/__init__.py", line 123, in __enter__ self.begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 508, in begin super(CephLab, self).begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 342, in begin self.execute_playbook() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 374, in execute_playbook self._handle_failure(command, status) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 402, in _handle_failure raise AnsibleFailedError(failures) teuthology.exceptions.AnsibleFailedError: [',', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at'] 2024-04-10T23:01:12.161 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=78bb7be407464d69ad1158fdaf04c7e5 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/__init__.py", line 123, in __enter__ self.begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 508, in begin super(CephLab, self).begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 342, in begin self.execute_playbook() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 374, in execute_playbook self._handle_failure(command, status) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 402, in _handle_failure raise AnsibleFailedError(failures) teuthology.exceptions.AnsibleFailedError: [',', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at'] 2024-04-10T23:01:12.163 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-10T23:01:12.171 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-10T23:01:12.171 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-10T23:01:12.179 DEBUG:teuthology.orchestra.run.smithi055:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-10T23:01:12.270 DEBUG:teuthology.orchestra.run.smithi116:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-10T23:01:12.340 DEBUG:teuthology.orchestra.run.smithi055:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T23:01:12.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T23:01:12.376 DEBUG:teuthology.orchestra.run.smithi116:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T23:01:12.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T23:01:12.408 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-10T23:01:12.416 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-10T23:01:12.423 INFO:teuthology.task.internal:Duration was 1042.434542 seconds 2024-04-10T23:01:12.423 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-10T23:01:12.431 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-10T23:01:12.431 DEBUG:teuthology.orchestra.run.smithi055:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-10T23:01:12.434 DEBUG:teuthology.orchestra.run.smithi116:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-10T23:01:12.490 INFO:teuthology.orchestra.run.smithi055.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T23:01:12.499 INFO:teuthology.orchestra.run.smithi116.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T23:01:12.602 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-10T23:01:12.603 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi055.front.sepia.ceph.com 2024-04-10T23:01:12.603 DEBUG:teuthology.orchestra.run.smithi055:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-10T23:01:12.631 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi116.front.sepia.ceph.com 2024-04-10T23:01:12.632 DEBUG:teuthology.orchestra.run.smithi116:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-10T23:01:12.654 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-10T23:01:12.655 DEBUG:teuthology.orchestra.run.smithi055:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-10T23:01:12.674 DEBUG:teuthology.orchestra.run.smithi116:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-10T23:01:12.735 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-10T23:01:12.736 DEBUG:teuthology.orchestra.run.smithi055:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-10T23:01:12.738 DEBUG:teuthology.orchestra.run.smithi116:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-10T23:01:12.841 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-10T23:01:12.849 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-10T23:01:12.849 DEBUG:teuthology.orchestra.run.smithi055:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-10T23:01:12.882 DEBUG:teuthology.orchestra.run.smithi116:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-10T23:01:12.908 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-10T23:01:12.917 DEBUG:teuthology.orchestra.run.smithi055:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-10T23:01:12.924 DEBUG:teuthology.orchestra.run.smithi116:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-10T23:01:12.955 INFO:teuthology.orchestra.run.smithi055.stdout:kernel.core_pattern = core 2024-04-10T23:01:12.974 INFO:teuthology.orchestra.run.smithi116.stdout:kernel.core_pattern = core 2024-04-10T23:01:13.031 DEBUG:teuthology.orchestra.run.smithi055:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-10T23:01:13.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T23:01:13.053 DEBUG:teuthology.orchestra.run.smithi116:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-10T23:01:13.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T23:01:13.088 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-10T23:01:13.097 INFO:teuthology.task.internal:Transferring archived files... 2024-04-10T23:01:13.099 DEBUG:teuthology.misc:Transferring archived files from smithi055:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650195/remote/smithi055 2024-04-10T23:01:13.100 DEBUG:teuthology.orchestra.run.smithi055:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-10T23:01:13.157 DEBUG:teuthology.misc:Transferring archived files from smithi116:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650195/remote/smithi116 2024-04-10T23:01:13.158 DEBUG:teuthology.orchestra.run.smithi116:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-10T23:01:13.212 INFO:teuthology.task.internal:Removing archive directory... 2024-04-10T23:01:13.212 DEBUG:teuthology.orchestra.run.smithi055:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-10T23:01:13.215 DEBUG:teuthology.orchestra.run.smithi116:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-10T23:01:13.263 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-10T23:01:13.273 INFO:teuthology.task.internal:Not uploading archives. 2024-04-10T23:01:13.273 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-10T23:01:13.282 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-10T23:01:13.282 DEBUG:teuthology.orchestra.run.smithi055:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-10T23:01:13.284 DEBUG:teuthology.orchestra.run.smithi116:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-10T23:01:13.303 INFO:teuthology.orchestra.run.smithi055.stdout: 262171 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 10 23:01 /home/ubuntu/cephtest 2024-04-10T23:01:13.320 INFO:teuthology.orchestra.run.smithi116.stdout: 262171 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 10 23:01 /home/ubuntu/cephtest 2024-04-10T23:01:13.322 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-10T23:01:13.330 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-10T23:01:13.357 INFO:teuthology.nuke:Checking targets against current locks 2024-04-10T23:01:13.422 DEBUG:teuthology.nuke:shortname: smithi055 2024-04-10T23:01:13.422 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T23:01:13.527 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi055.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650195', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:21.720259', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMPHzA4Wo1NnldvhC/y5ufPFUpgEiYnwZHiRSfM1A/wg91wGYf05SsIf56GtgYs/p+zUWOeBBOAfxn5K+816ams='} 2024-04-10T23:01:13.570 DEBUG:teuthology.nuke:shortname: smithi116 2024-04-10T23:01:13.570 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T23:01:13.615 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi116.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650195', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:21.721542', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFKILFoaEe+TTWRXVQWo8PI7J5YbBh/JQ+BHUCwIlG3GVpuyVw0DMZCCIV1g8Ehdx+KLXiGhEryPYLYPgA9YN7U='} 2024-04-10T23:01:13.683 INFO:teuthology.orchestra.console.smithi055:Power off 2024-04-10T23:01:13.683 DEBUG:teuthology.orchestra.console.smithi055:pexpect command: ipmitool -H smithi055.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-10T23:01:13.720 INFO:teuthology.orchestra.console.smithi116:Power off 2024-04-10T23:01:13.721 DEBUG:teuthology.orchestra.console.smithi116:pexpect command: ipmitool -H smithi116.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-10T23:01:13.738 DEBUG:teuthology.orchestra.console.smithi055:power off output: Chassis Power Control: Down/Off 2024-04-10T23:01:13.738 DEBUG:teuthology.orchestra.console.smithi055:pexpect command: ipmitool -H smithi055.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:01:13.777 DEBUG:teuthology.orchestra.console.smithi116:power off output: Chassis Power Control: Down/Off 2024-04-10T23:01:13.777 DEBUG:teuthology.orchestra.console.smithi116:pexpect command: ipmitool -H smithi116.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:01:13.794 DEBUG:teuthology.orchestra.console.smithi055:check power output: Chassis Power is on 2024-04-10T23:01:13.830 DEBUG:teuthology.orchestra.console.smithi116:check power output: Chassis Power is on 2024-04-10T23:01:17.795 DEBUG:teuthology.orchestra.console.smithi055:pexpect command: ipmitool -H smithi055.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:01:17.831 DEBUG:teuthology.orchestra.console.smithi116:pexpect command: ipmitool -H smithi116.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:01:17.906 DEBUG:teuthology.orchestra.console.smithi055:check power output: Chassis Power is on 2024-04-10T23:01:17.941 DEBUG:teuthology.orchestra.console.smithi116:check power output: Chassis Power is on 2024-04-10T23:01:21.911 DEBUG:teuthology.orchestra.console.smithi055:pexpect command: ipmitool -H smithi055.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:01:21.943 DEBUG:teuthology.orchestra.console.smithi116:pexpect command: ipmitool -H smithi116.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:01:22.023 DEBUG:teuthology.orchestra.console.smithi055:check power output: Chassis Power is off 2024-04-10T23:01:22.053 DEBUG:teuthology.orchestra.console.smithi116:check power output: Chassis Power is on 2024-04-10T23:01:22.124 INFO:teuthology.orchestra.console.smithi055:Power off completed 2024-04-10T23:01:26.057 DEBUG:teuthology.orchestra.console.smithi116:pexpect command: ipmitool -H smithi116.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T23:01:26.168 DEBUG:teuthology.orchestra.console.smithi116:check power output: Chassis Power is off 2024-04-10T23:01:26.268 INFO:teuthology.orchestra.console.smithi116:Power off completed 2024-04-10T23:01:26.374 INFO:teuthology.run:Summary data: description: upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} duration: 1042.4345421791077 failure_reason: '['','', ''Bugzilla. In case of problems, contact the owner of this repository.'', ''Enabling a Copr repository. Please note that this repository is not part'', ''Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable'', ''Please do not file bug reports about these packages in Fedora'', ''The Fedora Project does not exercise any power over the contents of'', ''and packages are not held to any quality or security level.'', ''of the main distribution, and quality may vary.'', ''this repository beyond the rules outlined in the Copr FAQ at'']' owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=78bb7be407464d69ad1158fdaf04c7e5 status: dead success: false 2024-04-10T23:01:26.375 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-10T23:01:26.463 INFO:teuthology.run:DEAD