2024-04-10T22:22:46.758 INFO:root:teuthology version: 0.0.1.dev267+g6c63784 2024-04-10T22:22:46.759 DEBUG:teuthology.run:Teuthology command: teuthology --description upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_rhel8 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} --archive /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650171 --owner scheduled_teuthology@teuthology --name teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi --verbose -- /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650171/orig.config.yaml 2024-04-10T22:22:46.814 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-10T22:22:46.898 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650171 branch: reef description: upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_rhel8 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} email: ceph-qa@ceph.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: pacific clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' first_in_suite: false job_id: '7650171' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'librbd C and C++ api tests ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run basic import/export cli tests for rbd ' name: teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: rhel os_version: '8.6' overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: reef sha1: 4f70a9a90881107996c8b7a3e89ba8b32624713c owner: scheduled_teuthology@teuthology priority: 830 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' seed: 4604 sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: pacific clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 0/1 suite: upgrade:pacific-x suite_branch: reef suite_path: /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 4f70a9a90881107996c8b7a3e89ba8b32624713c targets: smithi005.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKedV5JYzw892DdGl1jA1zalQ/8SV39jnaNhMLkSNcCQcZjiZvWsEocEcMXJgD4Pot+odyQqyaFbL40PJjmstUQ= smithi080.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBK3ZLCDplUTWysDIF0+pOvs4/dC2XNaGYGCp6DY6xoYYuy8PNMvwcF6Ih/6Lkvry/5AtL+LMmZT708WtuXonwQk= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools:rhel8 --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - install: branch: pacific exclude_packages: - ceph-volume - cephadm: cephadm_branch: pacific cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:pacific - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 6c637841c215537a4502385240412f1966e0faab timestamp: 2024-04-10_01:08:08 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-04-10T22:22:46.898 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa; will attempt to use it 2024-04-10T22:22:46.899 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa/tasks 2024-04-10T22:22:46.899 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-10T22:22:46.900 INFO:teuthology.task.internal:Checking packages... 2024-04-10T22:22:46.921 INFO:teuthology.task.internal:Checking packages for os_type 'rhel', flavor 'default' and ceph hash 'd540ebaca6b131a1dd560e7f69e024b133bbaa42' 2024-04-10T22:22:46.921 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-10T22:22:46.921 INFO:teuthology.packaging:ref: None 2024-04-10T22:22:46.921 INFO:teuthology.packaging:tag: None 2024-04-10T22:22:46.921 INFO:teuthology.packaging:branch: reef 2024-04-10T22:22:46.921 INFO:teuthology.packaging:sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 2024-04-10T22:22:46.921 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef 2024-04-10T22:22:47.315 INFO:teuthology.task.internal:Found packages for ceph version 18.2.2-1189.gc96c8d47 2024-04-10T22:22:47.316 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-10T22:22:47.323 INFO:teuthology.task.internal:no buildpackages task found 2024-04-10T22:22:47.324 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-10T22:22:47.345 INFO:teuthology.task.internal:Saving configuration 2024-04-10T22:22:47.366 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-10T22:22:47.372 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T22:22:47.397 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi005.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650171', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:15:49.694978', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKedV5JYzw892DdGl1jA1zalQ/8SV39jnaNhMLkSNcCQcZjiZvWsEocEcMXJgD4Pot+odyQqyaFbL40PJjmstUQ='} 2024-04-10T22:22:47.422 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi080.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650171', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:15:49.693649', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBK3ZLCDplUTWysDIF0+pOvs4/dC2XNaGYGCp6DY6xoYYuy8PNMvwcF6Ih/6Lkvry/5AtL+LMmZT708WtuXonwQk='} 2024-04-10T22:22:47.422 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-10T22:22:47.431 INFO:teuthology.task.internal:roles: ubuntu@smithi005.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-04-10T22:22:47.431 INFO:teuthology.task.internal:roles: ubuntu@smithi080.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-04-10T22:22:47.431 INFO:teuthology.run_tasks:Running task console_log... 2024-04-10T22:22:47.495 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f600a2e00d0>, signals=[15]) 2024-04-10T22:22:47.495 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-10T22:22:47.501 INFO:teuthology.task.internal:Opening connections... 2024-04-10T22:22:47.502 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi005.front.sepia.ceph.com 2024-04-10T22:22:47.503 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi005.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:22:47.578 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi080.front.sepia.ceph.com 2024-04-10T22:22:47.578 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi080.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:22:47.649 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-10T22:22:47.656 DEBUG:teuthology.orchestra.run.smithi005:> uname -m 2024-04-10T22:22:47.678 INFO:teuthology.orchestra.run.smithi005.stdout:x86_64 2024-04-10T22:22:47.679 DEBUG:teuthology.orchestra.run.smithi005:> cat /etc/os-release 2024-04-10T22:22:47.739 INFO:teuthology.orchestra.run.smithi005.stdout:NAME="Red Hat Enterprise Linux" 2024-04-10T22:22:47.740 INFO:teuthology.orchestra.run.smithi005.stdout:VERSION="8.6 (Ootpa)" 2024-04-10T22:22:47.740 INFO:teuthology.orchestra.run.smithi005.stdout:ID="rhel" 2024-04-10T22:22:47.740 INFO:teuthology.orchestra.run.smithi005.stdout:ID_LIKE="fedora" 2024-04-10T22:22:47.740 INFO:teuthology.orchestra.run.smithi005.stdout:VERSION_ID="8.6" 2024-04-10T22:22:47.740 INFO:teuthology.orchestra.run.smithi005.stdout:PLATFORM_ID="platform:el8" 2024-04-10T22:22:47.740 INFO:teuthology.orchestra.run.smithi005.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2024-04-10T22:22:47.740 INFO:teuthology.orchestra.run.smithi005.stdout:ANSI_COLOR="0;31" 2024-04-10T22:22:47.740 INFO:teuthology.orchestra.run.smithi005.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2024-04-10T22:22:47.740 INFO:teuthology.orchestra.run.smithi005.stdout:HOME_URL="https://www.redhat.com/" 2024-04-10T22:22:47.740 INFO:teuthology.orchestra.run.smithi005.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2024-04-10T22:22:47.740 INFO:teuthology.orchestra.run.smithi005.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-10T22:22:47.740 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-10T22:22:47.740 INFO:teuthology.orchestra.run.smithi005.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2024-04-10T22:22:47.740 INFO:teuthology.orchestra.run.smithi005.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2024-04-10T22:22:47.740 INFO:teuthology.orchestra.run.smithi005.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2024-04-10T22:22:47.741 INFO:teuthology.orchestra.run.smithi005.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2024-04-10T22:22:47.741 INFO:teuthology.lock.ops:Updating smithi005.front.sepia.ceph.com on lock server 2024-04-10T22:22:47.768 DEBUG:teuthology.orchestra.run.smithi080:> uname -m 2024-04-10T22:22:47.788 INFO:teuthology.orchestra.run.smithi080.stdout:x86_64 2024-04-10T22:22:47.788 DEBUG:teuthology.orchestra.run.smithi080:> cat /etc/os-release 2024-04-10T22:22:47.848 INFO:teuthology.orchestra.run.smithi080.stdout:NAME="Red Hat Enterprise Linux" 2024-04-10T22:22:47.848 INFO:teuthology.orchestra.run.smithi080.stdout:VERSION="8.6 (Ootpa)" 2024-04-10T22:22:47.848 INFO:teuthology.orchestra.run.smithi080.stdout:ID="rhel" 2024-04-10T22:22:47.848 INFO:teuthology.orchestra.run.smithi080.stdout:ID_LIKE="fedora" 2024-04-10T22:22:47.848 INFO:teuthology.orchestra.run.smithi080.stdout:VERSION_ID="8.6" 2024-04-10T22:22:47.848 INFO:teuthology.orchestra.run.smithi080.stdout:PLATFORM_ID="platform:el8" 2024-04-10T22:22:47.848 INFO:teuthology.orchestra.run.smithi080.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2024-04-10T22:22:47.848 INFO:teuthology.orchestra.run.smithi080.stdout:ANSI_COLOR="0;31" 2024-04-10T22:22:47.848 INFO:teuthology.orchestra.run.smithi080.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2024-04-10T22:22:47.848 INFO:teuthology.orchestra.run.smithi080.stdout:HOME_URL="https://www.redhat.com/" 2024-04-10T22:22:47.849 INFO:teuthology.orchestra.run.smithi080.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2024-04-10T22:22:47.849 INFO:teuthology.orchestra.run.smithi080.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-10T22:22:47.849 INFO:teuthology.orchestra.run.smithi080.stdout: 2024-04-10T22:22:47.849 INFO:teuthology.orchestra.run.smithi080.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2024-04-10T22:22:47.849 INFO:teuthology.orchestra.run.smithi080.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2024-04-10T22:22:47.849 INFO:teuthology.orchestra.run.smithi080.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2024-04-10T22:22:47.849 INFO:teuthology.orchestra.run.smithi080.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2024-04-10T22:22:47.849 INFO:teuthology.lock.ops:Updating smithi080.front.sepia.ceph.com on lock server 2024-04-10T22:22:47.867 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-10T22:22:47.876 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-10T22:22:47.881 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-10T22:22:47.881 DEBUG:teuthology.orchestra.run.smithi005:> test '!' -e /home/ubuntu/cephtest 2024-04-10T22:22:47.884 DEBUG:teuthology.orchestra.run.smithi080:> test '!' -e /home/ubuntu/cephtest 2024-04-10T22:22:47.907 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-10T22:22:47.973 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-10T22:22:47.974 DEBUG:teuthology.orchestra.run.smithi005:> test -z $(ls -A /var/lib/ceph) 2024-04-10T22:22:47.976 DEBUG:teuthology.orchestra.run.smithi080:> test -z $(ls -A /var/lib/ceph) 2024-04-10T22:22:48.002 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-10T22:22:48.058 INFO:teuthology.run_tasks:Running task kernel... 2024-04-10T22:22:48.072 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-04-10T22:22:48.072 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-04-10T22:22:48.072 DEBUG:teuthology.orchestra.run.smithi005:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-10T22:22:48.073 DEBUG:teuthology.orchestra.run.smithi080:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-10T22:22:48.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:22:48.094 DEBUG:teuthology.orchestra.run.smithi080:> uname -r 2024-04-10T22:22:48.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:22:48.095 DEBUG:teuthology.orchestra.run.smithi005:> uname -r 2024-04-10T22:22:48.154 INFO:teuthology.orchestra.run.smithi080.stdout:4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:22:48.155 INFO:teuthology.task.kernel:Running kernel on smithi080: 4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:22:48.155 DEBUG:teuthology.orchestra.run.smithi080:> sudo yum install -y kernel 2024-04-10T22:22:48.156 INFO:teuthology.orchestra.run.smithi005.stdout:4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:22:48.156 INFO:teuthology.task.kernel:Running kernel on smithi005: 4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:22:48.156 DEBUG:teuthology.orchestra.run.smithi005:> sudo yum install -y kernel 2024-04-10T22:22:49.642 INFO:teuthology.orchestra.run.smithi080.stdout:Updating Subscription Management repositories. 2024-04-10T22:22:49.643 INFO:teuthology.orchestra.run.smithi080.stdout:Unable to read consumer identity 2024-04-10T22:22:49.768 INFO:teuthology.orchestra.run.smithi080.stdout: 2024-04-10T22:22:49.769 INFO:teuthology.orchestra.run.smithi080.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:22:49.769 INFO:teuthology.orchestra.run.smithi080.stdout: 2024-04-10T22:22:49.830 INFO:teuthology.orchestra.run.smithi005.stdout:Updating Subscription Management repositories. 2024-04-10T22:22:49.830 INFO:teuthology.orchestra.run.smithi005.stdout:Unable to read consumer identity 2024-04-10T22:22:49.938 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-10T22:22:49.938 INFO:teuthology.orchestra.run.smithi005.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:22:49.938 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-10T22:23:31.847 INFO:teuthology.orchestra.run.smithi080.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:39 2024-04-10T22:23:31.848 INFO:teuthology.orchestra.run.smithi080.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:23:31.848 INFO:teuthology.orchestra.run.smithi080.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:23:31.848 INFO:teuthology.orchestra.run.smithi080.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:23:31.886 INFO:teuthology.orchestra.run.smithi005.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:40 2024-04-10T22:23:31.887 INFO:teuthology.orchestra.run.smithi005.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:23:31.887 INFO:teuthology.orchestra.run.smithi005.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:23:31.887 INFO:teuthology.orchestra.run.smithi005.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:25:42.653 INFO:teuthology.orchestra.run.smithi005.stdout:Extra Packages for Enterprise Linux 127 kB/s | 16 MB 02:10 2024-04-10T22:25:45.261 INFO:teuthology.orchestra.run.smithi080.stdout:Extra Packages for Enterprise Linux 125 kB/s | 16 MB 02:13 2024-04-10T22:25:46.636 INFO:teuthology.orchestra.run.smithi005.stdout:lab-extras 330 kB/s | 24 kB 00:00 2024-04-10T22:25:46.720 INFO:teuthology.orchestra.run.smithi005.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:25:47.397 INFO:teuthology.orchestra.run.smithi005.stdout:Package kernel-4.18.0-513.9.1.el8_9.x86_64 is already installed. 2024-04-10T22:25:47.397 INFO:teuthology.orchestra.run.smithi005.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2024-04-10T22:25:47.471 INFO:teuthology.orchestra.run.smithi005.stdout:Dependencies resolved. 2024-04-10T22:25:47.473 INFO:teuthology.orchestra.run.smithi005.stdout:Nothing to do. 2024-04-10T22:25:47.473 INFO:teuthology.orchestra.run.smithi005.stdout:Complete! 2024-04-10T22:25:47.612 DEBUG:teuthology.orchestra.run.smithi005:> echo no | sudo yum reinstall kernel || true 2024-04-10T22:25:48.058 INFO:teuthology.orchestra.run.smithi005.stdout:Updating Subscription Management repositories. 2024-04-10T22:25:48.058 INFO:teuthology.orchestra.run.smithi005.stdout:Unable to read consumer identity 2024-04-10T22:25:48.125 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-10T22:25:48.125 INFO:teuthology.orchestra.run.smithi005.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:25:48.125 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-10T22:25:48.842 INFO:teuthology.orchestra.run.smithi080.stdout:lab-extras 451 kB/s | 24 kB 00:00 2024-04-10T22:25:48.916 INFO:teuthology.orchestra.run.smithi080.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:25:49.470 INFO:teuthology.orchestra.run.smithi080.stdout:Package kernel-4.18.0-513.9.1.el8_9.x86_64 is already installed. 2024-04-10T22:25:49.470 INFO:teuthology.orchestra.run.smithi080.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2024-04-10T22:25:49.555 INFO:teuthology.orchestra.run.smithi080.stdout:Dependencies resolved. 2024-04-10T22:25:49.556 INFO:teuthology.orchestra.run.smithi080.stdout:Nothing to do. 2024-04-10T22:25:49.556 INFO:teuthology.orchestra.run.smithi080.stdout:Complete! 2024-04-10T22:25:49.712 DEBUG:teuthology.orchestra.run.smithi080:> echo no | sudo yum reinstall kernel || true 2024-04-10T22:25:50.051 INFO:teuthology.orchestra.run.smithi080.stdout:Updating Subscription Management repositories. 2024-04-10T22:25:50.051 INFO:teuthology.orchestra.run.smithi080.stdout:Unable to read consumer identity 2024-04-10T22:25:50.107 INFO:teuthology.orchestra.run.smithi080.stdout: 2024-04-10T22:25:50.107 INFO:teuthology.orchestra.run.smithi080.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:25:50.107 INFO:teuthology.orchestra.run.smithi080.stdout: 2024-04-10T22:26:34.583 INFO:teuthology.orchestra.run.smithi080.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:44 2024-04-10T22:26:34.584 INFO:teuthology.orchestra.run.smithi080.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:26:34.584 INFO:teuthology.orchestra.run.smithi080.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:26:34.584 INFO:teuthology.orchestra.run.smithi080.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:26:34.649 INFO:teuthology.orchestra.run.smithi005.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:46 2024-04-10T22:26:34.650 INFO:teuthology.orchestra.run.smithi005.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:26:34.650 INFO:teuthology.orchestra.run.smithi005.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:26:34.650 INFO:teuthology.orchestra.run.smithi005.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:26:34.733 INFO:teuthology.orchestra.run.smithi080.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:26:34.733 INFO:teuthology.orchestra.run.smithi080.stdout:Last metadata expiration check: 0:00:46 ago on Wed 10 Apr 2024 10:25:48 PM UTC. 2024-04-10T22:26:34.815 INFO:teuthology.orchestra.run.smithi005.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:26:34.815 INFO:teuthology.orchestra.run.smithi005.stdout:Last metadata expiration check: 0:00:48 ago on Wed 10 Apr 2024 10:25:46 PM UTC. 2024-04-10T22:26:34.833 INFO:teuthology.orchestra.run.smithi080.stdout:Installed package kernel-4.18.0-513.9.1.el8_9.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available. 2024-04-10T22:26:34.833 INFO:teuthology.orchestra.run.smithi080.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2024-04-10T22:26:34.876 INFO:teuthology.orchestra.run.smithi080.stderr:Error: No packages marked for reinstall. 2024-04-10T22:26:34.916 INFO:teuthology.orchestra.run.smithi005.stdout:Installed package kernel-4.18.0-513.9.1.el8_9.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available. 2024-04-10T22:26:34.917 INFO:teuthology.orchestra.run.smithi005.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2024-04-10T22:26:34.922 DEBUG:teuthology.orchestra.run.smithi080:> sudo yum reinstall -y kernel || true 2024-04-10T22:26:34.960 INFO:teuthology.orchestra.run.smithi005.stderr:Error: No packages marked for reinstall. 2024-04-10T22:26:35.016 DEBUG:teuthology.orchestra.run.smithi005:> sudo yum reinstall -y kernel || true 2024-04-10T22:26:35.257 INFO:teuthology.orchestra.run.smithi080.stdout:Updating Subscription Management repositories. 2024-04-10T22:26:35.258 INFO:teuthology.orchestra.run.smithi080.stdout:Unable to read consumer identity 2024-04-10T22:26:35.310 INFO:teuthology.orchestra.run.smithi080.stdout: 2024-04-10T22:26:35.310 INFO:teuthology.orchestra.run.smithi080.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:26:35.310 INFO:teuthology.orchestra.run.smithi080.stdout: 2024-04-10T22:26:35.399 INFO:teuthology.orchestra.run.smithi005.stdout:Updating Subscription Management repositories. 2024-04-10T22:26:35.399 INFO:teuthology.orchestra.run.smithi005.stdout:Unable to read consumer identity 2024-04-10T22:26:35.452 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-10T22:26:35.452 INFO:teuthology.orchestra.run.smithi005.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-10T22:26:35.452 INFO:teuthology.orchestra.run.smithi005.stdout: 2024-04-10T22:27:15.769 INFO:teuthology.orchestra.run.smithi005.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:40 2024-04-10T22:27:15.770 INFO:teuthology.orchestra.run.smithi005.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:27:15.770 INFO:teuthology.orchestra.run.smithi005.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:27:15.770 INFO:teuthology.orchestra.run.smithi005.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:27:15.774 INFO:teuthology.orchestra.run.smithi080.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:40 2024-04-10T22:27:15.775 INFO:teuthology.orchestra.run.smithi080.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:27:15.775 INFO:teuthology.orchestra.run.smithi080.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:27:15.775 INFO:teuthology.orchestra.run.smithi080.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:27:15.920 INFO:teuthology.orchestra.run.smithi080.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:27:15.921 INFO:teuthology.orchestra.run.smithi080.stdout:Last metadata expiration check: 0:01:27 ago on Wed 10 Apr 2024 10:25:48 PM UTC. 2024-04-10T22:27:15.929 INFO:teuthology.orchestra.run.smithi005.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:27:15.930 INFO:teuthology.orchestra.run.smithi005.stdout:Last metadata expiration check: 0:01:29 ago on Wed 10 Apr 2024 10:25:46 PM UTC. 2024-04-10T22:27:15.989 INFO:teuthology.orchestra.run.smithi080.stdout:Installed package kernel-4.18.0-513.9.1.el8_9.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available. 2024-04-10T22:27:15.990 INFO:teuthology.orchestra.run.smithi080.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2024-04-10T22:27:15.998 INFO:teuthology.orchestra.run.smithi005.stdout:Installed package kernel-4.18.0-513.9.1.el8_9.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available. 2024-04-10T22:27:15.998 INFO:teuthology.orchestra.run.smithi005.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2024-04-10T22:27:16.016 INFO:teuthology.orchestra.run.smithi080.stderr:Error: No packages marked for reinstall. 2024-04-10T22:27:16.028 INFO:teuthology.orchestra.run.smithi005.stderr:Error: No packages marked for reinstall. 2024-04-10T22:27:16.066 DEBUG:teuthology.orchestra.run.smithi080:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:27:16.087 DEBUG:teuthology.orchestra.run.smithi005:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:27:16.226 INFO:teuthology.orchestra.run.smithi080.stdout:kernel-4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:27:16.226 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:27:16.226 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-10T22:27:16.227 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-04-10T22:27:16.227 DEBUG:teuthology.orchestra.run.smithi080:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-10T22:27:16.262 INFO:teuthology.orchestra.run.smithi080.stdout:ttyS1 2024-04-10T22:27:16.266 INFO:teuthology.orchestra.run.smithi005.stdout:kernel-4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:27:16.267 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-513.9.1.el8_9.x86_64 2024-04-10T22:27:16.267 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-10T22:27:16.267 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-04-10T22:27:16.267 DEBUG:teuthology.orchestra.run.smithi005:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-10T22:27:16.279 DEBUG:teuthology.parallel:result is None 2024-04-10T22:27:16.305 INFO:teuthology.orchestra.run.smithi005.stdout:ttyS1 2024-04-10T22:27:16.326 DEBUG:teuthology.parallel:result is None 2024-04-10T22:27:16.326 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-10T22:27:16.335 INFO:teuthology.task.internal:Creating test directory... 2024-04-10T22:27:16.335 DEBUG:teuthology.orchestra.run.smithi005:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-10T22:27:16.369 DEBUG:teuthology.orchestra.run.smithi080:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-10T22:27:16.387 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-10T22:27:16.394 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-10T22:27:16.409 INFO:teuthology.task.internal:Creating archive directory... 2024-04-10T22:27:16.410 DEBUG:teuthology.orchestra.run.smithi005:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-10T22:27:16.430 DEBUG:teuthology.orchestra.run.smithi080:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-10T22:27:16.463 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-10T22:27:16.471 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-10T22:27:16.471 DEBUG:teuthology.orchestra.run.smithi005:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-10T22:27:16.500 DEBUG:teuthology.orchestra.run.smithi080:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-10T22:27:16.537 INFO:teuthology.orchestra.run.smithi080.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:27:16.544 INFO:teuthology.orchestra.run.smithi005.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:27:16.548 INFO:teuthology.orchestra.run.smithi080.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:27:16.558 INFO:teuthology.orchestra.run.smithi005.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:27:16.560 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-10T22:27:16.571 INFO:teuthology.task.internal:Configuring sudo... 2024-04-10T22:27:16.571 DEBUG:teuthology.orchestra.run.smithi005:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-10T22:27:16.574 DEBUG:teuthology.orchestra.run.smithi080:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-10T22:27:16.616 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-10T22:27:16.626 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-10T22:27:16.626 DEBUG:teuthology.orchestra.run.smithi005:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-10T22:27:16.650 DEBUG:teuthology.orchestra.run.smithi080:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-10T22:27:16.673 DEBUG:teuthology.orchestra.run.smithi005:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:27:16.757 DEBUG:teuthology.orchestra.run.smithi005:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:27:16.815 DEBUG:teuthology.orchestra.run.smithi005:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:27:16.906 DEBUG:teuthology.orchestra.run.smithi005:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:27:16.953 DEBUG:teuthology.orchestra.run.smithi005:> set -ex 2024-04-10T22:27:16.953 DEBUG:teuthology.orchestra.run.smithi005:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-10T22:27:17.028 DEBUG:teuthology.orchestra.run.smithi080:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:27:17.071 DEBUG:teuthology.orchestra.run.smithi080:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:27:17.120 DEBUG:teuthology.orchestra.run.smithi080:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:27:17.201 DEBUG:teuthology.orchestra.run.smithi080:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:27:17.246 DEBUG:teuthology.orchestra.run.smithi080:> set -ex 2024-04-10T22:27:17.246 DEBUG:teuthology.orchestra.run.smithi080:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-10T22:27:17.312 DEBUG:teuthology.orchestra.run.smithi005:> sudo service rsyslog restart 2024-04-10T22:27:17.316 DEBUG:teuthology.orchestra.run.smithi080:> sudo service rsyslog restart 2024-04-10T22:27:17.374 INFO:teuthology.orchestra.run.smithi005.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:27:17.427 INFO:teuthology.orchestra.run.smithi080.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:27:17.869 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-10T22:27:17.878 INFO:teuthology.task.internal:Starting timer... 2024-04-10T22:27:17.878 INFO:teuthology.run_tasks:Running task pcp... 2024-04-10T22:27:17.921 INFO:teuthology.run_tasks:Running task selinux... 2024-04-10T22:27:17.930 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-04-10T22:27:17.931 DEBUG:teuthology.orchestra.run.smithi005:> sudo service auditd rotate 2024-04-10T22:27:18.017 INFO:teuthology.orchestra.run.smithi005.stdout:Rotating logs: 2024-04-10T22:27:18.022 DEBUG:teuthology.orchestra.run.smithi080:> sudo service auditd rotate 2024-04-10T22:27:18.095 INFO:teuthology.orchestra.run.smithi080.stdout:Rotating logs: 2024-04-10T22:27:18.097 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-10T22:27:18.097 DEBUG:teuthology.orchestra.run.smithi005:> /usr/sbin/getenforce 2024-04-10T22:27:18.120 INFO:teuthology.orchestra.run.smithi005.stdout:Permissive 2024-04-10T22:27:18.120 DEBUG:teuthology.orchestra.run.smithi080:> /usr/sbin/getenforce 2024-04-10T22:27:18.144 INFO:teuthology.orchestra.run.smithi080.stdout:Permissive 2024-04-10T22:27:18.144 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi005.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi080.front.sepia.ceph.com': 'permissive'} 2024-04-10T22:27:18.145 DEBUG:teuthology.orchestra.run.smithi005:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:27:18.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:27:18.191 DEBUG:teuthology.orchestra.run.smithi080:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:27:18.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:27:18.218 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-10T22:27:18.218 DEBUG:teuthology.orchestra.run.smithi005:> sudo /usr/sbin/setenforce permissive 2024-04-10T22:27:18.262 DEBUG:teuthology.orchestra.run.smithi080:> sudo /usr/sbin/setenforce permissive 2024-04-10T22:27:18.289 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-10T22:27:18.300 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-04-10T22:27:18.304 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-04-10T22:27:18.430 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-10T22:27:18.450 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-10T22:27:18.451 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi005.front.sepia.ceph.com,smithi080.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-10T22:43:55.296 INFO:teuthology.task.ansible:Archiving ansible failure log at: /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650171/ansible_failures.yaml 2024-04-10T22:43:55.298 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/__init__.py", line 123, in __enter__ self.begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 508, in begin super(CephLab, self).begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 342, in begin self.execute_playbook() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 374, in execute_playbook self._handle_failure(command, status) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 402, in _handle_failure raise AnsibleFailedError(failures) teuthology.exceptions.AnsibleFailedError: [',', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at'] 2024-04-10T22:43:55.542 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=8d310ae07e044a52998f5f0e36088b59 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/__init__.py", line 123, in __enter__ self.begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 508, in begin super(CephLab, self).begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 342, in begin self.execute_playbook() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 374, in execute_playbook self._handle_failure(command, status) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 402, in _handle_failure raise AnsibleFailedError(failures) teuthology.exceptions.AnsibleFailedError: [',', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at'] 2024-04-10T22:43:55.545 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-10T22:43:55.553 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-10T22:43:55.553 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-10T22:43:55.563 DEBUG:teuthology.orchestra.run.smithi005:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-10T22:43:55.651 DEBUG:teuthology.orchestra.run.smithi080:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-10T22:43:55.717 DEBUG:teuthology.orchestra.run.smithi005:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:43:55.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:43:55.754 DEBUG:teuthology.orchestra.run.smithi080:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:43:55.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:43:55.785 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-10T22:43:55.795 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-10T22:43:55.839 INFO:teuthology.task.internal:Duration was 997.961539 seconds 2024-04-10T22:43:55.840 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-10T22:43:55.854 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-10T22:43:55.854 DEBUG:teuthology.orchestra.run.smithi005:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-10T22:43:55.857 DEBUG:teuthology.orchestra.run.smithi080:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-10T22:43:55.906 INFO:teuthology.orchestra.run.smithi080.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:43:55.913 INFO:teuthology.orchestra.run.smithi005.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:43:56.178 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-10T22:43:56.178 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi005.front.sepia.ceph.com 2024-04-10T22:43:56.178 DEBUG:teuthology.orchestra.run.smithi005:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-10T22:43:56.205 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi080.front.sepia.ceph.com 2024-04-10T22:43:56.206 DEBUG:teuthology.orchestra.run.smithi080:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-10T22:43:56.228 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-10T22:43:56.228 DEBUG:teuthology.orchestra.run.smithi005:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-10T22:43:56.249 DEBUG:teuthology.orchestra.run.smithi080:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-10T22:43:56.310 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-10T22:43:56.311 DEBUG:teuthology.orchestra.run.smithi005:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-10T22:43:56.313 DEBUG:teuthology.orchestra.run.smithi080:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-10T22:43:56.417 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-10T22:43:56.426 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-10T22:43:56.426 DEBUG:teuthology.orchestra.run.smithi005:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-10T22:43:56.460 DEBUG:teuthology.orchestra.run.smithi080:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-10T22:43:56.486 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-10T22:43:56.498 DEBUG:teuthology.orchestra.run.smithi005:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-10T22:43:56.502 DEBUG:teuthology.orchestra.run.smithi080:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-10T22:43:56.534 INFO:teuthology.orchestra.run.smithi005.stdout:kernel.core_pattern = core 2024-04-10T22:43:56.553 INFO:teuthology.orchestra.run.smithi080.stdout:kernel.core_pattern = core 2024-04-10T22:43:56.577 DEBUG:teuthology.orchestra.run.smithi005:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-10T22:43:56.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:43:56.623 DEBUG:teuthology.orchestra.run.smithi080:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-10T22:43:56.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:43:56.639 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-10T22:43:56.754 INFO:teuthology.task.internal:Transferring archived files... 2024-04-10T22:43:56.755 DEBUG:teuthology.misc:Transferring archived files from smithi005:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650171/remote/smithi005 2024-04-10T22:43:56.756 DEBUG:teuthology.orchestra.run.smithi005:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-10T22:43:56.818 DEBUG:teuthology.misc:Transferring archived files from smithi080:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650171/remote/smithi080 2024-04-10T22:43:56.819 DEBUG:teuthology.orchestra.run.smithi080:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-10T22:43:56.880 INFO:teuthology.task.internal:Removing archive directory... 2024-04-10T22:43:56.880 DEBUG:teuthology.orchestra.run.smithi005:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-10T22:43:56.883 DEBUG:teuthology.orchestra.run.smithi080:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-10T22:43:56.902 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-10T22:43:56.912 INFO:teuthology.task.internal:Not uploading archives. 2024-04-10T22:43:56.913 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-10T22:43:56.940 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-10T22:43:56.941 DEBUG:teuthology.orchestra.run.smithi005:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-10T22:43:56.945 DEBUG:teuthology.orchestra.run.smithi080:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-10T22:43:56.961 INFO:teuthology.orchestra.run.smithi080.stdout: 262171 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 10 22:43 /home/ubuntu/cephtest 2024-04-10T22:43:56.963 INFO:teuthology.orchestra.run.smithi005.stdout: 262171 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 10 22:43 /home/ubuntu/cephtest 2024-04-10T22:43:56.965 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-10T22:43:56.974 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-10T22:43:56.992 INFO:teuthology.nuke:Checking targets against current locks 2024-04-10T22:43:57.038 DEBUG:teuthology.nuke:shortname: smithi005 2024-04-10T22:43:57.038 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T22:43:57.110 DEBUG:teuthology.nuke:shortname: smithi080 2024-04-10T22:43:57.110 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T22:43:57.141 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi005.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650171', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:15:49.694978', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKedV5JYzw892DdGl1jA1zalQ/8SV39jnaNhMLkSNcCQcZjiZvWsEocEcMXJgD4Pot+odyQqyaFbL40PJjmstUQ='} 2024-04-10T22:43:57.195 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi080.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650171', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:15:49.693649', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBK3ZLCDplUTWysDIF0+pOvs4/dC2XNaGYGCp6DY6xoYYuy8PNMvwcF6Ih/6Lkvry/5AtL+LMmZT708WtuXonwQk='} 2024-04-10T22:43:57.282 INFO:teuthology.orchestra.console.smithi005:Power off 2024-04-10T22:43:57.282 DEBUG:teuthology.orchestra.console.smithi005:pexpect command: ipmitool -H smithi005.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-10T22:43:57.345 INFO:teuthology.orchestra.console.smithi080:Power off 2024-04-10T22:43:57.346 DEBUG:teuthology.orchestra.console.smithi080:pexpect command: ipmitool -H smithi080.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-10T22:43:57.355 DEBUG:teuthology.orchestra.console.smithi005:power off output: Chassis Power Control: Down/Off 2024-04-10T22:43:57.355 DEBUG:teuthology.orchestra.console.smithi005:pexpect command: ipmitool -H smithi005.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:43:57.415 DEBUG:teuthology.orchestra.console.smithi080:power off output: Chassis Power Control: Down/Off 2024-04-10T22:43:57.416 DEBUG:teuthology.orchestra.console.smithi080:pexpect command: ipmitool -H smithi080.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:43:57.424 DEBUG:teuthology.orchestra.console.smithi005:check power output: Chassis Power is on 2024-04-10T22:43:57.467 DEBUG:teuthology.orchestra.console.smithi080:check power output: Chassis Power is on 2024-04-10T22:44:01.425 DEBUG:teuthology.orchestra.console.smithi005:pexpect command: ipmitool -H smithi005.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:44:01.468 DEBUG:teuthology.orchestra.console.smithi080:pexpect command: ipmitool -H smithi080.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:44:01.537 DEBUG:teuthology.orchestra.console.smithi005:check power output: Chassis Power is on 2024-04-10T22:44:01.579 DEBUG:teuthology.orchestra.console.smithi080:check power output: Chassis Power is on 2024-04-10T22:44:05.542 DEBUG:teuthology.orchestra.console.smithi005:pexpect command: ipmitool -H smithi005.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:44:05.580 DEBUG:teuthology.orchestra.console.smithi080:pexpect command: ipmitool -H smithi080.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:44:05.653 DEBUG:teuthology.orchestra.console.smithi005:check power output: Chassis Power is off 2024-04-10T22:44:05.690 DEBUG:teuthology.orchestra.console.smithi080:check power output: Chassis Power is on 2024-04-10T22:44:05.754 INFO:teuthology.orchestra.console.smithi005:Power off completed 2024-04-10T22:44:09.694 DEBUG:teuthology.orchestra.console.smithi080:pexpect command: ipmitool -H smithi080.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:44:09.807 DEBUG:teuthology.orchestra.console.smithi080:check power output: Chassis Power is off 2024-04-10T22:44:09.907 INFO:teuthology.orchestra.console.smithi080:Power off completed 2024-04-10T22:44:10.014 INFO:teuthology.run:Summary data: description: upgrade:pacific-x/stress-split/{0-distro/rhel_8.6_container_tools_rhel8 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} duration: 997.9615390300751 failure_reason: '['','', ''Bugzilla. In case of problems, contact the owner of this repository.'', ''Enabling a Copr repository. Please note that this repository is not part'', ''Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable'', ''Please do not file bug reports about these packages in Fedora'', ''The Fedora Project does not exercise any power over the contents of'', ''and packages are not held to any quality or security level.'', ''of the main distribution, and quality may vary.'', ''this repository beyond the rules outlined in the Copr FAQ at'']' owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=8d310ae07e044a52998f5f0e36088b59 status: dead success: false 2024-04-10T22:44:10.015 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-10T22:44:10.115 INFO:teuthology.run:DEAD