2024-04-10T22:38:04.155 INFO:root:teuthology version: 0.0.1.dev267+g6c63784 2024-04-10T22:38:04.155 DEBUG:teuthology.run:Teuthology command: teuthology --name teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi --verbose --description upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_rhel8 0-roles 1-start 2-first-half-tasks/snaps-few-objects 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} --owner scheduled_teuthology@teuthology --archive /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650191 -- /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650191/orig.config.yaml 2024-04-10T22:38:04.216 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-10T22:38:04.308 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650191 branch: reef description: upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_rhel8 0-roles 1-start 2-first-half-tasks/snaps-few-objects 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} email: ceph-qa@ceph.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' first_in_suite: false job_id: '7650191' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run basic import/export cli tests for rbd ' name: teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: rhel os_version: '8.6' overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: reef sha1: 4f70a9a90881107996c8b7a3e89ba8b32624713c owner: scheduled_teuthology@teuthology priority: 830 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' seed: 4604 sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: pacific clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 0/1 suite: upgrade:pacific-x suite_branch: reef suite_path: /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 4f70a9a90881107996c8b7a3e89ba8b32624713c targets: smithi008.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMR0Ce1jd4El6mMvq2xKJ6tm2+SrJOEpRjg8VHd2HBPx03pI+k5CLnGh6hmfKDWd6I14GbbygARgsKLqBj122d0= smithi097.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNZ8AY5gmtj+u1t2fYJcvq8jctMbI2VcuVXebK4dcYXfMGt7RD0WFSWLaP5FLzoBpfAyud2GDpYSJv+y0LC73MY= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools:rhel8 --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - install: branch: pacific exclude_packages: - ceph-volume - cephadm: cephadm_branch: pacific cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:pacific - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 6c637841c215537a4502385240412f1966e0faab timestamp: 2024-04-10_01:08:08 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-04-10T22:38:04.308 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa; will attempt to use it 2024-04-10T22:38:04.309 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa/tasks 2024-04-10T22:38:04.309 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-10T22:38:04.311 INFO:teuthology.task.internal:Checking packages... 2024-04-10T22:38:04.336 INFO:teuthology.task.internal:Checking packages for os_type 'rhel', flavor 'default' and ceph hash 'd540ebaca6b131a1dd560e7f69e024b133bbaa42' 2024-04-10T22:38:04.336 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-10T22:38:04.336 INFO:teuthology.packaging:ref: None 2024-04-10T22:38:04.336 INFO:teuthology.packaging:tag: None 2024-04-10T22:38:04.336 INFO:teuthology.packaging:branch: reef 2024-04-10T22:38:04.336 INFO:teuthology.packaging:sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 2024-04-10T22:38:04.336 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef 2024-04-10T22:38:04.693 INFO:teuthology.task.internal:Found packages for ceph version 18.2.2-1189.gc96c8d47 2024-04-10T22:38:04.695 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-10T22:38:04.702 INFO:teuthology.task.internal:no buildpackages task found 2024-04-10T22:38:04.702 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-10T22:38:04.712 INFO:teuthology.task.internal:Saving configuration 2024-04-10T22:38:04.733 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-10T22:38:04.739 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T22:38:04.761 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi008.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650191', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:20.044123', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMR0Ce1jd4El6mMvq2xKJ6tm2+SrJOEpRjg8VHd2HBPx03pI+k5CLnGh6hmfKDWd6I14GbbygARgsKLqBj122d0='} 2024-04-10T22:38:04.784 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi097.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650191', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:20.042785', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNZ8AY5gmtj+u1t2fYJcvq8jctMbI2VcuVXebK4dcYXfMGt7RD0WFSWLaP5FLzoBpfAyud2GDpYSJv+y0LC73MY='} 2024-04-10T22:38:04.784 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-10T22:38:04.790 INFO:teuthology.task.internal:roles: ubuntu@smithi008.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-04-10T22:38:04.790 INFO:teuthology.task.internal:roles: ubuntu@smithi097.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-04-10T22:38:04.790 INFO:teuthology.run_tasks:Running task console_log... 2024-04-10T22:38:04.849 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fe54ccd40d0>, signals=[15]) 2024-04-10T22:38:04.850 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-10T22:38:04.856 INFO:teuthology.task.internal:Opening connections... 2024-04-10T22:38:04.856 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi008.front.sepia.ceph.com 2024-04-10T22:38:04.858 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi008.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:38:04.934 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi097.front.sepia.ceph.com 2024-04-10T22:38:04.935 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi097.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:38:05.005 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-10T22:38:05.011 DEBUG:teuthology.orchestra.run.smithi008:> uname -m 2024-04-10T22:38:05.033 INFO:teuthology.orchestra.run.smithi008.stdout:x86_64 2024-04-10T22:38:05.033 DEBUG:teuthology.orchestra.run.smithi008:> cat /etc/os-release 2024-04-10T22:38:05.093 INFO:teuthology.orchestra.run.smithi008.stdout:NAME="Red Hat Enterprise Linux" 2024-04-10T22:38:05.094 INFO:teuthology.orchestra.run.smithi008.stdout:VERSION="8.6 (Ootpa)" 2024-04-10T22:38:05.094 INFO:teuthology.orchestra.run.smithi008.stdout:ID="rhel" 2024-04-10T22:38:05.094 INFO:teuthology.orchestra.run.smithi008.stdout:ID_LIKE="fedora" 2024-04-10T22:38:05.094 INFO:teuthology.orchestra.run.smithi008.stdout:VERSION_ID="8.6" 2024-04-10T22:38:05.094 INFO:teuthology.orchestra.run.smithi008.stdout:PLATFORM_ID="platform:el8" 2024-04-10T22:38:05.094 INFO:teuthology.orchestra.run.smithi008.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2024-04-10T22:38:05.094 INFO:teuthology.orchestra.run.smithi008.stdout:ANSI_COLOR="0;31" 2024-04-10T22:38:05.094 INFO:teuthology.orchestra.run.smithi008.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2024-04-10T22:38:05.094 INFO:teuthology.orchestra.run.smithi008.stdout:HOME_URL="https://www.redhat.com/" 2024-04-10T22:38:05.094 INFO:teuthology.orchestra.run.smithi008.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2024-04-10T22:38:05.094 INFO:teuthology.orchestra.run.smithi008.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-10T22:38:05.094 INFO:teuthology.orchestra.run.smithi008.stdout: 2024-04-10T22:38:05.095 INFO:teuthology.orchestra.run.smithi008.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2024-04-10T22:38:05.095 INFO:teuthology.orchestra.run.smithi008.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2024-04-10T22:38:05.095 INFO:teuthology.orchestra.run.smithi008.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2024-04-10T22:38:05.095 INFO:teuthology.orchestra.run.smithi008.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2024-04-10T22:38:05.095 INFO:teuthology.lock.ops:Updating smithi008.front.sepia.ceph.com on lock server 2024-04-10T22:38:05.122 DEBUG:teuthology.orchestra.run.smithi097:> uname -m 2024-04-10T22:38:05.143 INFO:teuthology.orchestra.run.smithi097.stdout:x86_64 2024-04-10T22:38:05.143 DEBUG:teuthology.orchestra.run.smithi097:> cat /etc/os-release 2024-04-10T22:38:05.203 INFO:teuthology.orchestra.run.smithi097.stdout:NAME="Red Hat Enterprise Linux" 2024-04-10T22:38:05.203 INFO:teuthology.orchestra.run.smithi097.stdout:VERSION="8.6 (Ootpa)" 2024-04-10T22:38:05.203 INFO:teuthology.orchestra.run.smithi097.stdout:ID="rhel" 2024-04-10T22:38:05.203 INFO:teuthology.orchestra.run.smithi097.stdout:ID_LIKE="fedora" 2024-04-10T22:38:05.203 INFO:teuthology.orchestra.run.smithi097.stdout:VERSION_ID="8.6" 2024-04-10T22:38:05.203 INFO:teuthology.orchestra.run.smithi097.stdout:PLATFORM_ID="platform:el8" 2024-04-10T22:38:05.204 INFO:teuthology.orchestra.run.smithi097.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2024-04-10T22:38:05.204 INFO:teuthology.orchestra.run.smithi097.stdout:ANSI_COLOR="0;31" 2024-04-10T22:38:05.204 INFO:teuthology.orchestra.run.smithi097.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2024-04-10T22:38:05.204 INFO:teuthology.orchestra.run.smithi097.stdout:HOME_URL="https://www.redhat.com/" 2024-04-10T22:38:05.204 INFO:teuthology.orchestra.run.smithi097.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2024-04-10T22:38:05.204 INFO:teuthology.orchestra.run.smithi097.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-10T22:38:05.204 INFO:teuthology.orchestra.run.smithi097.stdout: 2024-04-10T22:38:05.204 INFO:teuthology.orchestra.run.smithi097.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2024-04-10T22:38:05.204 INFO:teuthology.orchestra.run.smithi097.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2024-04-10T22:38:05.204 INFO:teuthology.orchestra.run.smithi097.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2024-04-10T22:38:05.204 INFO:teuthology.orchestra.run.smithi097.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2024-04-10T22:38:05.204 INFO:teuthology.lock.ops:Updating smithi097.front.sepia.ceph.com on lock server 2024-04-10T22:38:05.222 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-10T22:38:05.232 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-10T22:38:05.237 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-10T22:38:05.238 DEBUG:teuthology.orchestra.run.smithi008:> test '!' -e /home/ubuntu/cephtest 2024-04-10T22:38:05.240 DEBUG:teuthology.orchestra.run.smithi097:> test '!' -e /home/ubuntu/cephtest 2024-04-10T22:38:05.262 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-10T22:38:05.268 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-10T22:38:05.268 DEBUG:teuthology.orchestra.run.smithi008:> test -z $(ls -A /var/lib/ceph) 2024-04-10T22:38:05.301 DEBUG:teuthology.orchestra.run.smithi097:> test -z $(ls -A /var/lib/ceph) 2024-04-10T22:38:05.332 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-10T22:38:05.382 INFO:teuthology.run_tasks:Running task kernel... 2024-04-10T22:38:05.397 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-04-10T22:38:05.398 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-04-10T22:38:05.398 DEBUG:teuthology.orchestra.run.smithi008:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-10T22:38:05.398 DEBUG:teuthology.orchestra.run.smithi097:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-10T22:38:05.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:38:05.417 DEBUG:teuthology.orchestra.run.smithi097:> uname -r 2024-04-10T22:38:05.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:38:05.418 DEBUG:teuthology.orchestra.run.smithi008:> uname -r 2024-04-10T22:38:05.477 INFO:teuthology.orchestra.run.smithi097.stdout:4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:38:05.477 INFO:teuthology.task.kernel:Running kernel on smithi097: 4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:38:05.478 DEBUG:teuthology.orchestra.run.smithi097:> sudo yum install -y kernel 2024-04-10T22:38:05.479 INFO:teuthology.orchestra.run.smithi008.stdout:4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:38:05.479 INFO:teuthology.task.kernel:Running kernel on smithi008: 4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:38:05.480 DEBUG:teuthology.orchestra.run.smithi008:> sudo yum install -y kernel 2024-04-10T22:38:07.014 INFO:teuthology.orchestra.run.smithi008.stdout:Updating Subscription Management repositories. 2024-04-10T22:38:07.015 INFO:teuthology.orchestra.run.smithi008.stdout:Unable to read consumer identity 2024-04-10T22:38:07.019 INFO:teuthology.orchestra.run.smithi097.stdout:Updating Subscription Management repositories. 2024-04-10T22:38:07.019 INFO:teuthology.orchestra.run.smithi097.stdout:Unable to read consumer identity 2024-04-10T22:38:07.121 INFO:teuthology.orchestra.run.smithi097.stdout: 2024-04-10T22:38:07.122 INFO:teuthology.orchestra.run.smithi097.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:38:07.122 INFO:teuthology.orchestra.run.smithi097.stdout: 2024-04-10T22:38:07.134 INFO:teuthology.orchestra.run.smithi008.stdout: 2024-04-10T22:38:07.134 INFO:teuthology.orchestra.run.smithi008.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:38:07.134 INFO:teuthology.orchestra.run.smithi008.stdout: 2024-04-10T22:38:55.005 INFO:teuthology.orchestra.run.smithi008.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:46 2024-04-10T22:38:55.006 INFO:teuthology.orchestra.run.smithi008.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:38:55.006 INFO:teuthology.orchestra.run.smithi008.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:38:55.006 INFO:teuthology.orchestra.run.smithi008.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:38:55.119 INFO:teuthology.orchestra.run.smithi097.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:46 2024-04-10T22:38:55.119 INFO:teuthology.orchestra.run.smithi097.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:38:55.119 INFO:teuthology.orchestra.run.smithi097.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:38:55.119 INFO:teuthology.orchestra.run.smithi097.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:38:58.918 INFO:teuthology.orchestra.run.smithi008.stdout:Extra Packages for Enterprise Linux 4.2 MB/s | 16 MB 00:03 2024-04-10T22:39:02.731 INFO:teuthology.orchestra.run.smithi097.stdout:Extra Packages for Enterprise Linux 2.1 MB/s | 16 MB 00:07 2024-04-10T22:39:02.884 INFO:teuthology.orchestra.run.smithi008.stdout:lab-extras 487 kB/s | 24 kB 00:00 2024-04-10T22:39:02.970 INFO:teuthology.orchestra.run.smithi008.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:39:03.562 INFO:teuthology.orchestra.run.smithi008.stdout:Package kernel-4.18.0-513.9.1.el8_9.x86_64 is already installed. 2024-04-10T22:39:03.562 INFO:teuthology.orchestra.run.smithi008.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2024-04-10T22:39:03.641 INFO:teuthology.orchestra.run.smithi008.stdout:Dependencies resolved. 2024-04-10T22:39:03.643 INFO:teuthology.orchestra.run.smithi008.stdout:Nothing to do. 2024-04-10T22:39:03.643 INFO:teuthology.orchestra.run.smithi008.stdout:Complete! 2024-04-10T22:39:03.804 DEBUG:teuthology.orchestra.run.smithi008:> echo no | sudo yum reinstall kernel || true 2024-04-10T22:39:04.191 INFO:teuthology.orchestra.run.smithi008.stdout:Updating Subscription Management repositories. 2024-04-10T22:39:04.191 INFO:teuthology.orchestra.run.smithi008.stdout:Unable to read consumer identity 2024-04-10T22:39:04.246 INFO:teuthology.orchestra.run.smithi008.stdout: 2024-04-10T22:39:04.246 INFO:teuthology.orchestra.run.smithi008.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:39:04.246 INFO:teuthology.orchestra.run.smithi008.stdout: 2024-04-10T22:39:06.314 INFO:teuthology.orchestra.run.smithi097.stdout:lab-extras 455 kB/s | 24 kB 00:00 2024-04-10T22:39:06.388 INFO:teuthology.orchestra.run.smithi097.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:39:07.034 INFO:teuthology.orchestra.run.smithi097.stdout:Package kernel-4.18.0-513.9.1.el8_9.x86_64 is already installed. 2024-04-10T22:39:07.034 INFO:teuthology.orchestra.run.smithi097.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2024-04-10T22:39:07.114 INFO:teuthology.orchestra.run.smithi097.stdout:Dependencies resolved. 2024-04-10T22:39:07.116 INFO:teuthology.orchestra.run.smithi097.stdout:Nothing to do. 2024-04-10T22:39:07.116 INFO:teuthology.orchestra.run.smithi097.stdout:Complete! 2024-04-10T22:39:07.236 DEBUG:teuthology.orchestra.run.smithi097:> echo no | sudo yum reinstall kernel || true 2024-04-10T22:39:07.612 INFO:teuthology.orchestra.run.smithi097.stdout:Updating Subscription Management repositories. 2024-04-10T22:39:07.612 INFO:teuthology.orchestra.run.smithi097.stdout:Unable to read consumer identity 2024-04-10T22:39:07.670 INFO:teuthology.orchestra.run.smithi097.stdout: 2024-04-10T22:39:07.670 INFO:teuthology.orchestra.run.smithi097.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:39:07.670 INFO:teuthology.orchestra.run.smithi097.stdout: 2024-04-10T22:39:45.536 INFO:teuthology.orchestra.run.smithi097.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:37 2024-04-10T22:39:45.537 INFO:teuthology.orchestra.run.smithi097.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:39:45.537 INFO:teuthology.orchestra.run.smithi097.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:39:45.537 INFO:teuthology.orchestra.run.smithi097.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:39:45.684 INFO:teuthology.orchestra.run.smithi097.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:39:45.685 INFO:teuthology.orchestra.run.smithi097.stdout:Last metadata expiration check: 0:00:39 ago on Wed 10 Apr 2024 10:39:06 PM UTC. 2024-04-10T22:39:45.775 INFO:teuthology.orchestra.run.smithi097.stdout:Installed package kernel-4.18.0-513.9.1.el8_9.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available. 2024-04-10T22:39:45.776 INFO:teuthology.orchestra.run.smithi097.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2024-04-10T22:39:45.820 INFO:teuthology.orchestra.run.smithi097.stderr:Error: No packages marked for reinstall. 2024-04-10T22:39:45.866 DEBUG:teuthology.orchestra.run.smithi097:> sudo yum reinstall -y kernel || true 2024-04-10T22:39:46.201 INFO:teuthology.orchestra.run.smithi097.stdout:Updating Subscription Management repositories. 2024-04-10T22:39:46.201 INFO:teuthology.orchestra.run.smithi097.stdout:Unable to read consumer identity 2024-04-10T22:39:46.257 INFO:teuthology.orchestra.run.smithi097.stdout: 2024-04-10T22:39:46.257 INFO:teuthology.orchestra.run.smithi097.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:39:46.257 INFO:teuthology.orchestra.run.smithi097.stdout: 2024-04-10T22:39:57.576 INFO:teuthology.orchestra.run.smithi008.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:53 2024-04-10T22:39:57.576 INFO:teuthology.orchestra.run.smithi008.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:39:57.576 INFO:teuthology.orchestra.run.smithi008.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:39:57.576 INFO:teuthology.orchestra.run.smithi008.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:39:57.744 INFO:teuthology.orchestra.run.smithi008.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:39:57.744 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:00:55 ago on Wed 10 Apr 2024 10:39:02 PM UTC. 2024-04-10T22:39:57.828 INFO:teuthology.orchestra.run.smithi008.stdout:Installed package kernel-4.18.0-513.9.1.el8_9.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available. 2024-04-10T22:39:57.832 INFO:teuthology.orchestra.run.smithi008.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2024-04-10T22:39:57.872 INFO:teuthology.orchestra.run.smithi008.stderr:Error: No packages marked for reinstall. 2024-04-10T22:39:57.926 DEBUG:teuthology.orchestra.run.smithi008:> sudo yum reinstall -y kernel || true 2024-04-10T22:39:58.308 INFO:teuthology.orchestra.run.smithi008.stdout:Updating Subscription Management repositories. 2024-04-10T22:39:58.309 INFO:teuthology.orchestra.run.smithi008.stdout:Unable to read consumer identity 2024-04-10T22:39:58.382 INFO:teuthology.orchestra.run.smithi008.stdout: 2024-04-10T22:39:58.383 INFO:teuthology.orchestra.run.smithi008.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:39:58.383 INFO:teuthology.orchestra.run.smithi008.stdout: 2024-04-10T22:40:25.730 INFO:teuthology.orchestra.run.smithi097.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:39 2024-04-10T22:40:25.731 INFO:teuthology.orchestra.run.smithi097.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:40:25.731 INFO:teuthology.orchestra.run.smithi097.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:40:25.731 INFO:teuthology.orchestra.run.smithi097.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:40:25.876 INFO:teuthology.orchestra.run.smithi097.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:40:25.876 INFO:teuthology.orchestra.run.smithi097.stdout:Last metadata expiration check: 0:01:19 ago on Wed 10 Apr 2024 10:39:06 PM UTC. 2024-04-10T22:40:25.944 INFO:teuthology.orchestra.run.smithi097.stdout:Installed package kernel-4.18.0-513.9.1.el8_9.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available. 2024-04-10T22:40:25.944 INFO:teuthology.orchestra.run.smithi097.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2024-04-10T22:40:25.979 INFO:teuthology.orchestra.run.smithi097.stderr:Error: No packages marked for reinstall. 2024-04-10T22:40:26.025 DEBUG:teuthology.orchestra.run.smithi097:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:40:26.183 INFO:teuthology.orchestra.run.smithi097.stdout:kernel-4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:40:26.183 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:40:26.183 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-10T22:40:26.183 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-04-10T22:40:26.184 DEBUG:teuthology.orchestra.run.smithi097:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-10T22:40:26.213 INFO:teuthology.orchestra.run.smithi097.stdout:ttyS1 2024-04-10T22:40:26.230 DEBUG:teuthology.parallel:result is None 2024-04-10T22:40:35.889 INFO:teuthology.orchestra.run.smithi008.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:37 2024-04-10T22:40:35.889 INFO:teuthology.orchestra.run.smithi008.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:40:35.889 INFO:teuthology.orchestra.run.smithi008.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:40:35.890 INFO:teuthology.orchestra.run.smithi008.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:40:36.048 INFO:teuthology.orchestra.run.smithi008.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:40:36.048 INFO:teuthology.orchestra.run.smithi008.stdout:Last metadata expiration check: 0:01:34 ago on Wed 10 Apr 2024 10:39:02 PM UTC. 2024-04-10T22:40:36.123 INFO:teuthology.orchestra.run.smithi008.stdout:Installed package kernel-4.18.0-513.9.1.el8_9.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available. 2024-04-10T22:40:36.123 INFO:teuthology.orchestra.run.smithi008.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2024-04-10T22:40:36.151 INFO:teuthology.orchestra.run.smithi008.stderr:Error: No packages marked for reinstall. 2024-04-10T22:40:36.206 DEBUG:teuthology.orchestra.run.smithi008:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:40:36.384 INFO:teuthology.orchestra.run.smithi008.stdout:kernel-4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:40:36.384 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:40:36.384 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-10T22:40:36.384 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-04-10T22:40:36.385 DEBUG:teuthology.orchestra.run.smithi008:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-10T22:40:36.419 INFO:teuthology.orchestra.run.smithi008.stdout:ttyS1 2024-04-10T22:40:36.432 DEBUG:teuthology.parallel:result is None 2024-04-10T22:40:36.433 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-10T22:40:36.442 INFO:teuthology.task.internal:Creating test directory... 2024-04-10T22:40:36.442 DEBUG:teuthology.orchestra.run.smithi008:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-10T22:40:36.475 DEBUG:teuthology.orchestra.run.smithi097:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-10T22:40:36.495 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-10T22:40:36.502 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-10T22:40:36.512 INFO:teuthology.task.internal:Creating archive directory... 2024-04-10T22:40:36.512 DEBUG:teuthology.orchestra.run.smithi008:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-10T22:40:36.535 DEBUG:teuthology.orchestra.run.smithi097:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-10T22:40:36.569 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-10T22:40:36.577 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-10T22:40:36.577 DEBUG:teuthology.orchestra.run.smithi008:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-10T22:40:36.602 DEBUG:teuthology.orchestra.run.smithi097:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-10T22:40:36.641 INFO:teuthology.orchestra.run.smithi097.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:40:36.645 INFO:teuthology.orchestra.run.smithi008.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:40:36.653 INFO:teuthology.orchestra.run.smithi097.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:40:36.658 INFO:teuthology.orchestra.run.smithi008.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:40:36.660 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-10T22:40:36.667 INFO:teuthology.task.internal:Configuring sudo... 2024-04-10T22:40:36.667 DEBUG:teuthology.orchestra.run.smithi008:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-10T22:40:36.669 DEBUG:teuthology.orchestra.run.smithi097:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-10T22:40:36.723 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-10T22:40:36.733 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-10T22:40:36.733 DEBUG:teuthology.orchestra.run.smithi008:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-10T22:40:36.743 DEBUG:teuthology.orchestra.run.smithi097:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-10T22:40:36.780 DEBUG:teuthology.orchestra.run.smithi008:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:40:36.852 DEBUG:teuthology.orchestra.run.smithi008:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:40:36.917 DEBUG:teuthology.orchestra.run.smithi008:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:40:36.964 DEBUG:teuthology.orchestra.run.smithi008:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:40:37.015 DEBUG:teuthology.orchestra.run.smithi008:> set -ex 2024-04-10T22:40:37.015 DEBUG:teuthology.orchestra.run.smithi008:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-10T22:40:37.091 DEBUG:teuthology.orchestra.run.smithi097:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:40:37.136 DEBUG:teuthology.orchestra.run.smithi097:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:40:37.195 DEBUG:teuthology.orchestra.run.smithi097:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:40:37.279 DEBUG:teuthology.orchestra.run.smithi097:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:40:37.324 DEBUG:teuthology.orchestra.run.smithi097:> set -ex 2024-04-10T22:40:37.324 DEBUG:teuthology.orchestra.run.smithi097:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-10T22:40:37.391 DEBUG:teuthology.orchestra.run.smithi008:> sudo service rsyslog restart 2024-04-10T22:40:37.394 DEBUG:teuthology.orchestra.run.smithi097:> sudo service rsyslog restart 2024-04-10T22:40:37.453 INFO:teuthology.orchestra.run.smithi008.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:40:37.482 INFO:teuthology.orchestra.run.smithi097.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:40:37.956 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-10T22:40:37.964 INFO:teuthology.task.internal:Starting timer... 2024-04-10T22:40:37.964 INFO:teuthology.run_tasks:Running task pcp... 2024-04-10T22:40:37.975 INFO:teuthology.run_tasks:Running task selinux... 2024-04-10T22:40:37.983 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-04-10T22:40:37.983 DEBUG:teuthology.orchestra.run.smithi008:> sudo service auditd rotate 2024-04-10T22:40:38.053 INFO:teuthology.orchestra.run.smithi008.stdout:Rotating logs: 2024-04-10T22:40:38.055 DEBUG:teuthology.orchestra.run.smithi097:> sudo service auditd rotate 2024-04-10T22:40:38.126 INFO:teuthology.orchestra.run.smithi097.stdout:Rotating logs: 2024-04-10T22:40:38.128 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-10T22:40:38.128 DEBUG:teuthology.orchestra.run.smithi008:> /usr/sbin/getenforce 2024-04-10T22:40:38.166 INFO:teuthology.orchestra.run.smithi008.stdout:Permissive 2024-04-10T22:40:38.166 DEBUG:teuthology.orchestra.run.smithi097:> /usr/sbin/getenforce 2024-04-10T22:40:38.187 INFO:teuthology.orchestra.run.smithi097.stdout:Permissive 2024-04-10T22:40:38.188 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi008.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi097.front.sepia.ceph.com': 'permissive'} 2024-04-10T22:40:38.188 DEBUG:teuthology.orchestra.run.smithi008:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:40:38.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:40:38.237 DEBUG:teuthology.orchestra.run.smithi097:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:40:38.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:40:38.264 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-10T22:40:38.264 DEBUG:teuthology.orchestra.run.smithi008:> sudo /usr/sbin/setenforce permissive 2024-04-10T22:40:38.308 DEBUG:teuthology.orchestra.run.smithi097:> sudo /usr/sbin/setenforce permissive 2024-04-10T22:40:38.355 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-10T22:40:38.366 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-10T22:40:38.383 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-10T22:40:38.384 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi008.front.sepia.ceph.com,smithi097.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-10T22:57:35.025 INFO:teuthology.task.ansible:Archiving ansible failure log at: /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650191/ansible_failures.yaml 2024-04-10T22:57:35.027 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/__init__.py", line 123, in __enter__ self.begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 508, in begin super(CephLab, self).begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 342, in begin self.execute_playbook() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 374, in execute_playbook self._handle_failure(command, status) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 402, in _handle_failure raise AnsibleFailedError(failures) teuthology.exceptions.AnsibleFailedError: [',', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at'] 2024-04-10T22:57:35.255 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=6c8cf32cae54429bae2317f035bf21ac Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/__init__.py", line 123, in __enter__ self.begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 508, in begin super(CephLab, self).begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 342, in begin self.execute_playbook() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 374, in execute_playbook self._handle_failure(command, status) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 402, in _handle_failure raise AnsibleFailedError(failures) teuthology.exceptions.AnsibleFailedError: [',', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at'] 2024-04-10T22:57:35.257 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-10T22:57:35.266 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-10T22:57:35.267 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-10T22:57:35.275 DEBUG:teuthology.orchestra.run.smithi008:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-10T22:57:35.362 DEBUG:teuthology.orchestra.run.smithi097:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-10T22:57:35.428 DEBUG:teuthology.orchestra.run.smithi008:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:57:35.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:57:35.464 DEBUG:teuthology.orchestra.run.smithi097:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:57:35.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:57:35.497 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-10T22:57:35.505 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-10T22:57:35.513 INFO:teuthology.task.internal:Duration was 1017.548881 seconds 2024-04-10T22:57:35.513 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-10T22:57:35.522 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-10T22:57:35.522 DEBUG:teuthology.orchestra.run.smithi008:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-10T22:57:35.525 DEBUG:teuthology.orchestra.run.smithi097:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-10T22:57:35.581 INFO:teuthology.orchestra.run.smithi008.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:57:35.587 INFO:teuthology.orchestra.run.smithi097.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:57:35.705 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-10T22:57:35.706 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi008.front.sepia.ceph.com 2024-04-10T22:57:35.706 DEBUG:teuthology.orchestra.run.smithi008:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-10T22:57:35.733 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi097.front.sepia.ceph.com 2024-04-10T22:57:35.734 DEBUG:teuthology.orchestra.run.smithi097:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-10T22:57:35.756 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-10T22:57:35.756 DEBUG:teuthology.orchestra.run.smithi008:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-10T22:57:35.776 DEBUG:teuthology.orchestra.run.smithi097:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-10T22:57:35.839 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-10T22:57:35.839 DEBUG:teuthology.orchestra.run.smithi008:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-10T22:57:35.842 DEBUG:teuthology.orchestra.run.smithi097:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-10T22:57:35.959 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-10T22:57:35.968 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-10T22:57:35.968 DEBUG:teuthology.orchestra.run.smithi008:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-10T22:57:36.001 DEBUG:teuthology.orchestra.run.smithi097:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-10T22:57:36.028 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-10T22:57:36.036 DEBUG:teuthology.orchestra.run.smithi008:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-10T22:57:36.044 DEBUG:teuthology.orchestra.run.smithi097:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-10T22:57:36.076 INFO:teuthology.orchestra.run.smithi008.stdout:kernel.core_pattern = core 2024-04-10T22:57:36.094 INFO:teuthology.orchestra.run.smithi097.stdout:kernel.core_pattern = core 2024-04-10T22:57:36.111 DEBUG:teuthology.orchestra.run.smithi008:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-10T22:57:36.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:57:36.165 DEBUG:teuthology.orchestra.run.smithi097:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-10T22:57:36.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:57:36.181 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-10T22:57:36.191 INFO:teuthology.task.internal:Transferring archived files... 2024-04-10T22:57:36.192 DEBUG:teuthology.misc:Transferring archived files from smithi008:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650191/remote/smithi008 2024-04-10T22:57:36.193 DEBUG:teuthology.orchestra.run.smithi008:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-10T22:57:36.270 DEBUG:teuthology.misc:Transferring archived files from smithi097:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650191/remote/smithi097 2024-04-10T22:57:36.271 DEBUG:teuthology.orchestra.run.smithi097:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-10T22:57:36.327 INFO:teuthology.task.internal:Removing archive directory... 2024-04-10T22:57:36.328 DEBUG:teuthology.orchestra.run.smithi008:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-10T22:57:36.330 DEBUG:teuthology.orchestra.run.smithi097:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-10T22:57:36.349 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-10T22:57:36.362 INFO:teuthology.task.internal:Not uploading archives. 2024-04-10T22:57:36.362 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-10T22:57:36.371 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-10T22:57:36.371 DEBUG:teuthology.orchestra.run.smithi008:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-10T22:57:36.392 DEBUG:teuthology.orchestra.run.smithi097:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-10T22:57:36.408 INFO:teuthology.orchestra.run.smithi097.stdout: 262171 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 10 22:57 /home/ubuntu/cephtest 2024-04-10T22:57:36.410 INFO:teuthology.orchestra.run.smithi008.stdout: 262171 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 10 22:57 /home/ubuntu/cephtest 2024-04-10T22:57:36.412 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-10T22:57:36.421 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-10T22:57:36.440 INFO:teuthology.nuke:Checking targets against current locks 2024-04-10T22:57:36.478 DEBUG:teuthology.nuke:shortname: smithi008 2024-04-10T22:57:36.479 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T22:57:36.527 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi008.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650191', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:20.044123', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMR0Ce1jd4El6mMvq2xKJ6tm2+SrJOEpRjg8VHd2HBPx03pI+k5CLnGh6hmfKDWd6I14GbbygARgsKLqBj122d0='} 2024-04-10T22:57:36.528 DEBUG:teuthology.nuke:shortname: smithi097 2024-04-10T22:57:36.528 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T22:57:36.575 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi097.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650191', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:31:20.042785', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNZ8AY5gmtj+u1t2fYJcvq8jctMbI2VcuVXebK4dcYXfMGt7RD0WFSWLaP5FLzoBpfAyud2GDpYSJv+y0LC73MY='} 2024-04-10T22:57:36.665 INFO:teuthology.orchestra.console.smithi008:Power off 2024-04-10T22:57:36.665 DEBUG:teuthology.orchestra.console.smithi008:pexpect command: ipmitool -H smithi008.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-10T22:57:36.703 INFO:teuthology.orchestra.console.smithi097:Power off 2024-04-10T22:57:36.703 DEBUG:teuthology.orchestra.console.smithi097:pexpect command: ipmitool -H smithi097.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-10T22:57:36.716 DEBUG:teuthology.orchestra.console.smithi008:power off output: Chassis Power Control: Down/Off 2024-04-10T22:57:36.717 DEBUG:teuthology.orchestra.console.smithi008:pexpect command: ipmitool -H smithi008.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:57:36.752 DEBUG:teuthology.orchestra.console.smithi097:power off output: Chassis Power Control: Down/Off 2024-04-10T22:57:36.753 DEBUG:teuthology.orchestra.console.smithi097:pexpect command: ipmitool -H smithi097.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:57:36.767 DEBUG:teuthology.orchestra.console.smithi008:check power output: Chassis Power is on 2024-04-10T22:57:36.807 DEBUG:teuthology.orchestra.console.smithi097:check power output: Chassis Power is on 2024-04-10T22:57:40.769 DEBUG:teuthology.orchestra.console.smithi008:pexpect command: ipmitool -H smithi008.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:57:40.808 DEBUG:teuthology.orchestra.console.smithi097:pexpect command: ipmitool -H smithi097.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:57:40.880 DEBUG:teuthology.orchestra.console.smithi008:check power output: Chassis Power is on 2024-04-10T22:57:40.919 DEBUG:teuthology.orchestra.console.smithi097:check power output: Chassis Power is on 2024-04-10T22:57:44.884 DEBUG:teuthology.orchestra.console.smithi008:pexpect command: ipmitool -H smithi008.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:57:44.919 DEBUG:teuthology.orchestra.console.smithi097:pexpect command: ipmitool -H smithi097.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:57:44.995 DEBUG:teuthology.orchestra.console.smithi008:check power output: Chassis Power is off 2024-04-10T22:57:45.030 DEBUG:teuthology.orchestra.console.smithi097:check power output: Chassis Power is off 2024-04-10T22:57:45.098 INFO:teuthology.orchestra.console.smithi008:Power off completed 2024-04-10T22:57:45.130 INFO:teuthology.orchestra.console.smithi097:Power off completed 2024-04-10T22:57:45.237 INFO:teuthology.run:Summary data: description: upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_rhel8 0-roles 1-start 2-first-half-tasks/snaps-few-objects 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} duration: 1017.5488810539246 failure_reason: '['','', ''Bugzilla. In case of problems, contact the owner of this repository.'', ''Enabling a Copr repository. Please note that this repository is not part'', ''Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable'', ''Please do not file bug reports about these packages in Fedora'', ''The Fedora Project does not exercise any power over the contents of'', ''and packages are not held to any quality or security level.'', ''of the main distribution, and quality may vary.'', ''this repository beyond the rules outlined in the Copr FAQ at'']' owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=6c8cf32cae54429bae2317f035bf21ac status: dead success: false 2024-04-10T22:57:45.237 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-10T22:57:45.331 INFO:teuthology.run:DEAD