2024-04-10T22:22:24.316 INFO:root:teuthology version: 0.0.1.dev267+g6c63784 2024-04-10T22:22:24.316 DEBUG:teuthology.run:Teuthology command: teuthology --name teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi --owner scheduled_teuthology@teuthology --archive /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650168 --description upgrade:pacific-x/stress-split/{0-distro/centos_8.stream_container_tools 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} --verbose -- /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650168/orig.config.yaml 2024-04-10T22:22:24.375 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-10T22:22:24.461 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650168 branch: reef description: upgrade:pacific-x/stress-split/{0-distro/centos_8.stream_container_tools 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} email: ceph-qa@ceph.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' first_in_suite: false job_id: '7650168' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 8.stream overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: reef sha1: 4f70a9a90881107996c8b7a3e89ba8b32624713c owner: scheduled_teuthology@teuthology priority: 830 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 4604 sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: pacific clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 0/1 suite: upgrade:pacific-x suite_branch: reef suite_path: /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 4f70a9a90881107996c8b7a3e89ba8b32624713c targets: smithi059.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBP+sK/wAJLkoKgXQ1X30YFTzsciU1igIUuEVDD0kUsNtkRAfaunV5ewSK2d/BABRFGO2x5G+7aZLjo94mxQg5tQ= smithi161.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBC3mPrcHfcLTa62/Rn/UXZQjZgaS1xhzzPZ8bedaU4ow+YLLPvZ7+BxF3dgrdrF8mJdTc5Xb31sOq2va0Dup3Xc= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - install: branch: pacific exclude_packages: - ceph-volume - cephadm: cephadm_branch: pacific cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:pacific - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 6c637841c215537a4502385240412f1966e0faab timestamp: 2024-04-10_01:08:08 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-04-10T22:22:24.462 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa; will attempt to use it 2024-04-10T22:22:24.462 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_4f70a9a90881107996c8b7a3e89ba8b32624713c/qa/tasks 2024-04-10T22:22:24.462 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-10T22:22:24.464 INFO:teuthology.task.internal:Checking packages... 2024-04-10T22:22:24.486 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'd540ebaca6b131a1dd560e7f69e024b133bbaa42' 2024-04-10T22:22:24.486 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-10T22:22:24.486 INFO:teuthology.packaging:ref: None 2024-04-10T22:22:24.486 INFO:teuthology.packaging:tag: None 2024-04-10T22:22:24.486 INFO:teuthology.packaging:branch: reef 2024-04-10T22:22:24.486 INFO:teuthology.packaging:sha1: d540ebaca6b131a1dd560e7f69e024b133bbaa42 2024-04-10T22:22:24.487 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef 2024-04-10T22:22:24.808 INFO:teuthology.task.internal:Found packages for ceph version 18.2.2-1189.gc96c8d47 2024-04-10T22:22:24.810 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-10T22:22:24.817 INFO:teuthology.task.internal:no buildpackages task found 2024-04-10T22:22:24.817 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-10T22:22:24.849 INFO:teuthology.task.internal:Saving configuration 2024-04-10T22:22:24.872 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-10T22:22:24.880 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T22:22:24.904 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi059.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650168', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:15:48.419823', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBP+sK/wAJLkoKgXQ1X30YFTzsciU1igIUuEVDD0kUsNtkRAfaunV5ewSK2d/BABRFGO2x5G+7aZLjo94mxQg5tQ='} 2024-04-10T22:22:24.932 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi161.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650168', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:15:48.421615', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBC3mPrcHfcLTa62/Rn/UXZQjZgaS1xhzzPZ8bedaU4ow+YLLPvZ7+BxF3dgrdrF8mJdTc5Xb31sOq2va0Dup3Xc='} 2024-04-10T22:22:24.932 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-10T22:22:24.940 INFO:teuthology.task.internal:roles: ubuntu@smithi059.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-04-10T22:22:24.940 INFO:teuthology.task.internal:roles: ubuntu@smithi161.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-04-10T22:22:24.940 INFO:teuthology.run_tasks:Running task console_log... 2024-04-10T22:22:25.043 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f7b080750d0>, signals=[15]) 2024-04-10T22:22:25.044 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-10T22:22:25.051 INFO:teuthology.task.internal:Opening connections... 2024-04-10T22:22:25.051 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi059.front.sepia.ceph.com 2024-04-10T22:22:25.053 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi059.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:22:25.131 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi161.front.sepia.ceph.com 2024-04-10T22:22:25.132 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi161.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:22:25.202 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-10T22:22:25.210 DEBUG:teuthology.orchestra.run.smithi059:> uname -m 2024-04-10T22:22:25.232 INFO:teuthology.orchestra.run.smithi059.stdout:x86_64 2024-04-10T22:22:25.232 DEBUG:teuthology.orchestra.run.smithi059:> cat /etc/os-release 2024-04-10T22:22:25.293 INFO:teuthology.orchestra.run.smithi059.stdout:NAME="CentOS Stream" 2024-04-10T22:22:25.293 INFO:teuthology.orchestra.run.smithi059.stdout:VERSION="8" 2024-04-10T22:22:25.293 INFO:teuthology.orchestra.run.smithi059.stdout:ID="centos" 2024-04-10T22:22:25.293 INFO:teuthology.orchestra.run.smithi059.stdout:ID_LIKE="rhel fedora" 2024-04-10T22:22:25.293 INFO:teuthology.orchestra.run.smithi059.stdout:VERSION_ID="8" 2024-04-10T22:22:25.293 INFO:teuthology.orchestra.run.smithi059.stdout:PLATFORM_ID="platform:el8" 2024-04-10T22:22:25.294 INFO:teuthology.orchestra.run.smithi059.stdout:PRETTY_NAME="CentOS Stream 8" 2024-04-10T22:22:25.294 INFO:teuthology.orchestra.run.smithi059.stdout:ANSI_COLOR="0;31" 2024-04-10T22:22:25.294 INFO:teuthology.orchestra.run.smithi059.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2024-04-10T22:22:25.294 INFO:teuthology.orchestra.run.smithi059.stdout:HOME_URL="https://centos.org/" 2024-04-10T22:22:25.294 INFO:teuthology.orchestra.run.smithi059.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-10T22:22:25.294 INFO:teuthology.orchestra.run.smithi059.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2024-04-10T22:22:25.294 INFO:teuthology.orchestra.run.smithi059.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-10T22:22:25.295 INFO:teuthology.lock.ops:Updating smithi059.front.sepia.ceph.com on lock server 2024-04-10T22:22:25.329 DEBUG:teuthology.orchestra.run.smithi161:> uname -m 2024-04-10T22:22:25.352 INFO:teuthology.orchestra.run.smithi161.stdout:x86_64 2024-04-10T22:22:25.352 DEBUG:teuthology.orchestra.run.smithi161:> cat /etc/os-release 2024-04-10T22:22:25.413 INFO:teuthology.orchestra.run.smithi161.stdout:NAME="CentOS Stream" 2024-04-10T22:22:25.413 INFO:teuthology.orchestra.run.smithi161.stdout:VERSION="8" 2024-04-10T22:22:25.413 INFO:teuthology.orchestra.run.smithi161.stdout:ID="centos" 2024-04-10T22:22:25.413 INFO:teuthology.orchestra.run.smithi161.stdout:ID_LIKE="rhel fedora" 2024-04-10T22:22:25.413 INFO:teuthology.orchestra.run.smithi161.stdout:VERSION_ID="8" 2024-04-10T22:22:25.413 INFO:teuthology.orchestra.run.smithi161.stdout:PLATFORM_ID="platform:el8" 2024-04-10T22:22:25.413 INFO:teuthology.orchestra.run.smithi161.stdout:PRETTY_NAME="CentOS Stream 8" 2024-04-10T22:22:25.413 INFO:teuthology.orchestra.run.smithi161.stdout:ANSI_COLOR="0;31" 2024-04-10T22:22:25.413 INFO:teuthology.orchestra.run.smithi161.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2024-04-10T22:22:25.414 INFO:teuthology.orchestra.run.smithi161.stdout:HOME_URL="https://centos.org/" 2024-04-10T22:22:25.414 INFO:teuthology.orchestra.run.smithi161.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-10T22:22:25.414 INFO:teuthology.orchestra.run.smithi161.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2024-04-10T22:22:25.414 INFO:teuthology.orchestra.run.smithi161.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-10T22:22:25.414 INFO:teuthology.lock.ops:Updating smithi161.front.sepia.ceph.com on lock server 2024-04-10T22:22:25.445 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-10T22:22:25.460 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-10T22:22:25.482 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-10T22:22:25.483 DEBUG:teuthology.orchestra.run.smithi059:> test '!' -e /home/ubuntu/cephtest 2024-04-10T22:22:25.485 DEBUG:teuthology.orchestra.run.smithi161:> test '!' -e /home/ubuntu/cephtest 2024-04-10T22:22:25.505 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-10T22:22:25.518 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-10T22:22:25.519 DEBUG:teuthology.orchestra.run.smithi059:> test -z $(ls -A /var/lib/ceph) 2024-04-10T22:22:25.546 DEBUG:teuthology.orchestra.run.smithi161:> test -z $(ls -A /var/lib/ceph) 2024-04-10T22:22:25.580 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-10T22:22:25.638 INFO:teuthology.run_tasks:Running task kernel... 2024-04-10T22:22:25.658 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-04-10T22:22:25.658 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-04-10T22:22:25.658 DEBUG:teuthology.orchestra.run.smithi059:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-10T22:22:25.658 DEBUG:teuthology.orchestra.run.smithi161:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-10T22:22:25.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:22:25.679 DEBUG:teuthology.orchestra.run.smithi059:> uname -r 2024-04-10T22:22:25.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:22:25.680 DEBUG:teuthology.orchestra.run.smithi161:> uname -r 2024-04-10T22:22:25.740 INFO:teuthology.orchestra.run.smithi059.stdout:4.18.0-489.el8.x86_64 2024-04-10T22:22:25.740 INFO:teuthology.task.kernel:Running kernel on smithi059: 4.18.0-489.el8.x86_64 2024-04-10T22:22:25.740 DEBUG:teuthology.orchestra.run.smithi059:> sudo yum install -y kernel 2024-04-10T22:22:25.741 INFO:teuthology.orchestra.run.smithi161.stdout:4.18.0-489.el8.x86_64 2024-04-10T22:22:25.741 INFO:teuthology.task.kernel:Running kernel on smithi161: 4.18.0-489.el8.x86_64 2024-04-10T22:22:25.741 DEBUG:teuthology.orchestra.run.smithi161:> sudo yum install -y kernel 2024-04-10T22:22:30.195 INFO:teuthology.orchestra.run.smithi161.stdout:CentOS Stream 8 - AppStream 16 MB/s | 28 MB 00:01 2024-04-10T22:24:39.275 INFO:teuthology.orchestra.run.smithi059.stdout:CentOS Stream 8 - AppStream 218 kB/s | 28 MB 02:10 2024-04-10T22:24:45.692 INFO:teuthology.orchestra.run.smithi059.stdout:CentOS Stream 8 - BaseOS 9.4 MB/s | 10 MB 00:01 2024-04-10T22:24:47.045 INFO:teuthology.orchestra.run.smithi161.stdout:CentOS Stream 8 - BaseOS 80 kB/s | 10 MB 02:11 2024-04-10T22:25:13.689 INFO:teuthology.orchestra.run.smithi059.stdout:CentOS Stream 8 - Extras 693 B/s | 18 kB 00:26 2024-04-10T22:25:14.140 INFO:teuthology.orchestra.run.smithi059.stdout:CentOS Stream 8 - Extras common packages 23 kB/s | 7.7 kB 00:00 2024-04-10T22:25:50.730 INFO:teuthology.orchestra.run.smithi059.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:36 2024-04-10T22:25:50.731 INFO:teuthology.orchestra.run.smithi059.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:25:50.731 INFO:teuthology.orchestra.run.smithi059.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:25:50.732 INFO:teuthology.orchestra.run.smithi059.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:25:53.838 INFO:teuthology.orchestra.run.smithi059.stdout:Extra Packages for Enterprise Linux 5.2 MB/s | 16 MB 00:03 2024-04-10T22:25:57.888 INFO:teuthology.orchestra.run.smithi059.stdout:lab-extras 313 kB/s | 24 kB 00:00 2024-04-10T22:25:58.039 INFO:teuthology.orchestra.run.smithi059.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:26:01.076 INFO:teuthology.orchestra.run.smithi059.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2024-04-10T22:26:01.077 INFO:teuthology.orchestra.run.smithi059.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2024-04-10T22:26:01.204 INFO:teuthology.orchestra.run.smithi059.stdout:Dependencies resolved. 2024-04-10T22:26:01.205 INFO:teuthology.orchestra.run.smithi059.stdout:================================================================================ 2024-04-10T22:26:01.205 INFO:teuthology.orchestra.run.smithi059.stdout: Package Architecture Version Repository Size 2024-04-10T22:26:01.205 INFO:teuthology.orchestra.run.smithi059.stdout:================================================================================ 2024-04-10T22:26:01.205 INFO:teuthology.orchestra.run.smithi059.stdout:Installing: 2024-04-10T22:26:01.206 INFO:teuthology.orchestra.run.smithi059.stdout: kernel x86_64 4.18.0-552.el8 baseos 10 M 2024-04-10T22:26:01.206 INFO:teuthology.orchestra.run.smithi059.stdout:Installing dependencies: 2024-04-10T22:26:01.206 INFO:teuthology.orchestra.run.smithi059.stdout: kernel-core x86_64 4.18.0-552.el8 baseos 44 M 2024-04-10T22:26:01.206 INFO:teuthology.orchestra.run.smithi059.stdout: kernel-modules x86_64 4.18.0-552.el8 baseos 36 M 2024-04-10T22:26:01.206 INFO:teuthology.orchestra.run.smithi059.stdout: 2024-04-10T22:26:01.206 INFO:teuthology.orchestra.run.smithi059.stdout:Transaction Summary 2024-04-10T22:26:01.206 INFO:teuthology.orchestra.run.smithi059.stdout:================================================================================ 2024-04-10T22:26:01.206 INFO:teuthology.orchestra.run.smithi059.stdout:Install 3 Packages 2024-04-10T22:26:01.206 INFO:teuthology.orchestra.run.smithi059.stdout: 2024-04-10T22:26:01.207 INFO:teuthology.orchestra.run.smithi059.stdout:Total download size: 90 M 2024-04-10T22:26:01.207 INFO:teuthology.orchestra.run.smithi059.stdout:Installed size: 96 M 2024-04-10T22:26:01.207 INFO:teuthology.orchestra.run.smithi059.stdout:Downloading Packages: 2024-04-10T22:26:05.698 INFO:teuthology.orchestra.run.smithi059.stdout:(1/3): kernel-4.18.0-552.el8.x86_64.rpm 2.4 MB/s | 10 MB 00:04 2024-04-10T22:26:07.906 INFO:teuthology.orchestra.run.smithi059.stdout:(2/3): kernel-modules-4.18.0-552.el8.x86_64.rpm 5.6 MB/s | 36 MB 00:06 2024-04-10T22:26:12.506 INFO:teuthology.orchestra.run.smithi059.stdout:(3/3): kernel-core-4.18.0-552.el8.x86_64.rpm 3.9 MB/s | 44 MB 00:11 2024-04-10T22:26:12.506 INFO:teuthology.orchestra.run.smithi059.stdout:-------------------------------------------------------------------------------- 2024-04-10T22:26:12.506 INFO:teuthology.orchestra.run.smithi059.stdout:Total 8.0 MB/s | 90 MB 00:11 2024-04-10T22:26:13.226 INFO:teuthology.orchestra.run.smithi059.stdout:Running transaction check 2024-04-10T22:26:13.678 INFO:teuthology.orchestra.run.smithi059.stdout:Transaction check succeeded. 2024-04-10T22:26:13.678 INFO:teuthology.orchestra.run.smithi059.stdout:Running transaction test 2024-04-10T22:26:14.981 INFO:teuthology.orchestra.run.smithi059.stdout:Transaction test succeeded. 2024-04-10T22:26:15.001 INFO:teuthology.orchestra.run.smithi059.stdout:Running transaction 2024-04-10T22:26:17.938 INFO:teuthology.orchestra.run.smithi059.stdout: Preparing : 1/1 2024-04-10T22:26:18.509 INFO:teuthology.orchestra.run.smithi059.stdout: Installing : kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:26:22.320 INFO:teuthology.orchestra.run.smithi059.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:26:22.707 INFO:teuthology.orchestra.run.smithi059.stdout: Installing : kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:26:26.415 INFO:teuthology.orchestra.run.smithi059.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:26:26.704 INFO:teuthology.orchestra.run.smithi059.stdout: Installing : kernel-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:26:50.047 INFO:teuthology.orchestra.run.smithi059.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:26:50.072 INFO:teuthology.orchestra.run.smithi059.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:26:50.431 INFO:teuthology.orchestra.run.smithi059.stdout: Running scriptlet: kernel-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:26:50.432 INFO:teuthology.orchestra.run.smithi059.stdout: Verifying : kernel-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:26:50.432 INFO:teuthology.orchestra.run.smithi059.stdout: Verifying : kernel-core-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:26:51.061 INFO:teuthology.orchestra.run.smithi059.stdout: Verifying : kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:26:51.061 INFO:teuthology.orchestra.run.smithi059.stdout: 2024-04-10T22:26:51.061 INFO:teuthology.orchestra.run.smithi059.stdout:Installed: 2024-04-10T22:26:51.061 INFO:teuthology.orchestra.run.smithi059.stdout: kernel-4.18.0-552.el8.x86_64 kernel-core-4.18.0-552.el8.x86_64 2024-04-10T22:26:51.061 INFO:teuthology.orchestra.run.smithi059.stdout: kernel-modules-4.18.0-552.el8.x86_64 2024-04-10T22:26:51.062 INFO:teuthology.orchestra.run.smithi059.stdout: 2024-04-10T22:26:51.062 INFO:teuthology.orchestra.run.smithi059.stdout:Complete! 2024-04-10T22:26:51.204 DEBUG:teuthology.orchestra.run.smithi059:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:26:51.435 INFO:teuthology.orchestra.run.smithi059.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-10T22:26:51.436 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-10T22:26:51.436 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-552.el8.x86_64 2024-04-10T22:26:51.436 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-10T22:26:51.436 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-10T22:26:51.436 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-04-10T22:26:51.436 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi059.front.sepia.ceph.com, path=None, version=distro) 2024-04-10T22:26:51.437 DEBUG:teuthology.orchestra.run.smithi059:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:26:51.657 INFO:teuthology.orchestra.run.smithi059.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-10T22:26:51.657 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-10T22:26:51.658 DEBUG:teuthology.orchestra.run.smithi059:> sudo rpm -qi grub2-tools 2024-04-10T22:26:51.692 INFO:teuthology.orchestra.run.smithi059.stdout:Name : grub2-tools 2024-04-10T22:26:51.692 INFO:teuthology.orchestra.run.smithi059.stdout:Epoch : 1 2024-04-10T22:26:51.692 INFO:teuthology.orchestra.run.smithi059.stdout:Version : 2.02 2024-04-10T22:26:51.692 INFO:teuthology.orchestra.run.smithi059.stdout:Release : 129.el8 2024-04-10T22:26:51.692 INFO:teuthology.orchestra.run.smithi059.stdout:Architecture: x86_64 2024-04-10T22:26:51.692 INFO:teuthology.orchestra.run.smithi059.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2024-04-10T22:26:51.692 INFO:teuthology.orchestra.run.smithi059.stdout:Group : System Environment/Base 2024-04-10T22:26:51.692 INFO:teuthology.orchestra.run.smithi059.stdout:Size : 9265586 2024-04-10T22:26:51.692 INFO:teuthology.orchestra.run.smithi059.stdout:License : GPLv3+ 2024-04-10T22:26:51.693 INFO:teuthology.orchestra.run.smithi059.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2024-04-10T22:26:51.693 INFO:teuthology.orchestra.run.smithi059.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2024-04-10T22:26:51.693 INFO:teuthology.orchestra.run.smithi059.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2024-04-10T22:26:51.693 INFO:teuthology.orchestra.run.smithi059.stdout:Build Host : kbuilder.bsys.centos.org 2024-04-10T22:26:51.693 INFO:teuthology.orchestra.run.smithi059.stdout:Relocations : (not relocatable) 2024-04-10T22:26:51.693 INFO:teuthology.orchestra.run.smithi059.stdout:Packager : CentOS BuildSystem 2024-04-10T22:26:51.693 INFO:teuthology.orchestra.run.smithi059.stdout:Vendor : CentOS 2024-04-10T22:26:51.693 INFO:teuthology.orchestra.run.smithi059.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-10T22:26:51.693 INFO:teuthology.orchestra.run.smithi059.stdout:Summary : Support tools for GRUB. 2024-04-10T22:26:51.693 INFO:teuthology.orchestra.run.smithi059.stdout:Description : 2024-04-10T22:26:51.693 INFO:teuthology.orchestra.run.smithi059.stdout: 2024-04-10T22:26:51.693 INFO:teuthology.orchestra.run.smithi059.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-10T22:26:51.694 INFO:teuthology.orchestra.run.smithi059.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-10T22:26:51.694 INFO:teuthology.orchestra.run.smithi059.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-10T22:26:51.694 INFO:teuthology.orchestra.run.smithi059.stdout:hardware devices. 2024-04-10T22:26:51.694 INFO:teuthology.orchestra.run.smithi059.stdout: 2024-04-10T22:26:51.694 INFO:teuthology.orchestra.run.smithi059.stdout:This subpackage provides tools for support of all platforms. 2024-04-10T22:26:51.695 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-10T22:26:51.695 INFO:teuthology.task.kernel:Updating grub on smithi059 to boot 4.18.0-552.el8.x86_64 2024-04-10T22:26:51.695 DEBUG:teuthology.orchestra.run.smithi059:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-10T22:26:53.003 INFO:teuthology.orchestra.run.smithi059.stderr:Generating grub configuration file ... 2024-04-10T22:26:55.204 INFO:teuthology.orchestra.run.smithi059.stderr:done 2024-04-10T22:26:55.205 DEBUG:teuthology.orchestra.run.smithi059:> mktemp 2024-04-10T22:26:55.235 INFO:teuthology.orchestra.run.smithi059.stdout:/tmp/tmp.UOlVunEWcY 2024-04-10T22:26:55.235 DEBUG:teuthology.orchestra.run.smithi059:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.UOlVunEWcY 2024-04-10T22:26:55.308 DEBUG:teuthology.orchestra.run.smithi059:> sudo chmod 0666 /tmp/tmp.UOlVunEWcY 2024-04-10T22:26:55.452 DEBUG:teuthology.orchestra.remote:smithi059:/tmp/tmp.UOlVunEWcY is 6KB 2024-04-10T22:26:55.501 DEBUG:teuthology.orchestra.run.smithi059:> rm -fr /tmp/tmp.UOlVunEWcY 2024-04-10T22:26:55.519 DEBUG:teuthology.orchestra.run.smithi059:> sudo /bin/ls /boot/loader/entries || true 2024-04-10T22:26:55.591 INFO:teuthology.orchestra.run.smithi059.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2024-04-10T22:26:55.592 INFO:teuthology.orchestra.run.smithi059.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2024-04-10T22:26:55.592 INFO:teuthology.orchestra.run.smithi059.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2024-04-10T22:26:55.592 INFO:teuthology.orchestra.run.smithi059.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64.conf 2024-04-10T22:26:55.593 DEBUG:teuthology.orchestra.run.smithi059:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64 2024-04-10T22:26:55.742 DEBUG:teuthology.orchestra.run.smithi059:> sudo shutdown -r now 2024-04-10T22:26:57.868 INFO:teuthology.orchestra.run.smithi161.stdout:CentOS Stream 8 - Extras 141 B/s | 18 kB 02:09 2024-04-10T22:26:58.310 INFO:teuthology.orchestra.run.smithi161.stdout:CentOS Stream 8 - Extras common packages 23 kB/s | 7.7 kB 00:00 2024-04-10T22:27:25.745 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-04-10T22:27:25.746 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi059.front.sepia.ceph.com' 2024-04-10T22:27:25.747 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi059.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:27:37.885 INFO:teuthology.orchestra.run.smithi161.stdout:Copr repo for python3-asyncssh owned by ceph 0.0 B/s | 0 B 00:39 2024-04-10T22:27:37.886 INFO:teuthology.orchestra.run.smithi161.stderr:Errors during downloading metadata for repository 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': 2024-04-10T22:27:37.886 INFO:teuthology.orchestra.run.smithi161.stderr: - Curl error (6): Couldn't resolve host name for https://download.copr.fedorainfracloud.org/results/ceph/python3-asyncssh/epel-8-x86_64/repodata/repomd.xml [Could not resolve host: download.copr.fedorainfracloud.org] 2024-04-10T22:27:37.886 INFO:teuthology.orchestra.run.smithi161.stderr:Error: Failed to download metadata for repo 'copr:copr.fedorainfracloud.org:ceph:python3-asyncssh': Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried 2024-04-10T22:27:44.290 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.59 2024-04-10T22:27:47.023 INFO:teuthology.orchestra.run.smithi161.stdout:Extra Packages for Enterprise Linux 1.8 MB/s | 16 MB 00:09 2024-04-10T22:27:50.773 INFO:teuthology.orchestra.run.smithi161.stdout:lab-extras 390 kB/s | 24 kB 00:00 2024-04-10T22:27:50.913 INFO:teuthology.orchestra.run.smithi161.stderr:Ignoring repositories: copr:copr.fedorainfracloud.org:ceph:python3-asyncssh 2024-04-10T22:27:53.292 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi059.front.sepia.ceph.com' 2024-04-10T22:27:53.293 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi059.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:27:53.787 INFO:teuthology.orchestra.run.smithi161.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2024-04-10T22:27:53.787 INFO:teuthology.orchestra.run.smithi161.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2024-04-10T22:27:53.917 INFO:teuthology.orchestra.run.smithi161.stdout:Dependencies resolved. 2024-04-10T22:27:53.917 INFO:teuthology.orchestra.run.smithi161.stdout:================================================================================ 2024-04-10T22:27:53.917 INFO:teuthology.orchestra.run.smithi161.stdout: Package Architecture Version Repository Size 2024-04-10T22:27:53.917 INFO:teuthology.orchestra.run.smithi161.stdout:================================================================================ 2024-04-10T22:27:53.918 INFO:teuthology.orchestra.run.smithi161.stdout:Installing: 2024-04-10T22:27:53.918 INFO:teuthology.orchestra.run.smithi161.stdout: kernel x86_64 4.18.0-552.el8 baseos 10 M 2024-04-10T22:27:53.918 INFO:teuthology.orchestra.run.smithi161.stdout:Installing dependencies: 2024-04-10T22:27:53.918 INFO:teuthology.orchestra.run.smithi161.stdout: kernel-core x86_64 4.18.0-552.el8 baseos 44 M 2024-04-10T22:27:53.918 INFO:teuthology.orchestra.run.smithi161.stdout: kernel-modules x86_64 4.18.0-552.el8 baseos 36 M 2024-04-10T22:27:53.918 INFO:teuthology.orchestra.run.smithi161.stdout: 2024-04-10T22:27:53.918 INFO:teuthology.orchestra.run.smithi161.stdout:Transaction Summary 2024-04-10T22:27:53.918 INFO:teuthology.orchestra.run.smithi161.stdout:================================================================================ 2024-04-10T22:27:53.918 INFO:teuthology.orchestra.run.smithi161.stdout:Install 3 Packages 2024-04-10T22:27:53.918 INFO:teuthology.orchestra.run.smithi161.stdout: 2024-04-10T22:27:53.919 INFO:teuthology.orchestra.run.smithi161.stdout:Total download size: 90 M 2024-04-10T22:27:53.919 INFO:teuthology.orchestra.run.smithi161.stdout:Installed size: 96 M 2024-04-10T22:27:53.919 INFO:teuthology.orchestra.run.smithi161.stdout:Downloading Packages: 2024-04-10T22:27:55.916 INFO:teuthology.orchestra.run.smithi161.stdout:(1/3): kernel-4.18.0-552.el8.x86_64.rpm 5.8 MB/s | 10 MB 00:01 2024-04-10T22:28:03.532 INFO:teuthology.orchestra.run.smithi161.stdout:(2/3): kernel-modules-4.18.0-552.el8.x86_64.rpm 3.9 MB/s | 36 MB 00:09 2024-04-10T22:28:06.848 INFO:teuthology.orchestra.run.smithi161.stdout:(3/3): kernel-core-4.18.0-552.el8.x86_64.rpm 3.4 MB/s | 44 MB 00:12 2024-04-10T22:28:06.848 INFO:teuthology.orchestra.run.smithi161.stdout:-------------------------------------------------------------------------------- 2024-04-10T22:28:06.848 INFO:teuthology.orchestra.run.smithi161.stdout:Total 7.0 MB/s | 90 MB 00:12 2024-04-10T22:28:07.533 INFO:teuthology.orchestra.run.smithi161.stdout:Running transaction check 2024-04-10T22:28:07.995 INFO:teuthology.orchestra.run.smithi161.stdout:Transaction check succeeded. 2024-04-10T22:28:07.995 INFO:teuthology.orchestra.run.smithi161.stdout:Running transaction test 2024-04-10T22:28:09.260 INFO:teuthology.orchestra.run.smithi161.stdout:Transaction test succeeded. 2024-04-10T22:28:09.288 INFO:teuthology.orchestra.run.smithi161.stdout:Running transaction 2024-04-10T22:28:12.245 INFO:teuthology.orchestra.run.smithi161.stdout: Preparing : 1/1 2024-04-10T22:28:12.787 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:28:16.458 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:28:16.845 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:28:20.290 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:28:20.567 INFO:teuthology.orchestra.run.smithi161.stdout: Installing : kernel-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:28:42.685 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: kernel-core-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:28:42.709 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:28:43.178 INFO:teuthology.orchestra.run.smithi161.stdout: Running scriptlet: kernel-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:28:43.179 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : kernel-4.18.0-552.el8.x86_64 1/3 2024-04-10T22:28:43.179 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : kernel-core-4.18.0-552.el8.x86_64 2/3 2024-04-10T22:28:43.664 INFO:teuthology.orchestra.run.smithi161.stdout: Verifying : kernel-modules-4.18.0-552.el8.x86_64 3/3 2024-04-10T22:28:43.664 INFO:teuthology.orchestra.run.smithi161.stdout: 2024-04-10T22:28:43.664 INFO:teuthology.orchestra.run.smithi161.stdout:Installed: 2024-04-10T22:28:43.664 INFO:teuthology.orchestra.run.smithi161.stdout: kernel-4.18.0-552.el8.x86_64 kernel-core-4.18.0-552.el8.x86_64 2024-04-10T22:28:43.665 INFO:teuthology.orchestra.run.smithi161.stdout: kernel-modules-4.18.0-552.el8.x86_64 2024-04-10T22:28:43.665 INFO:teuthology.orchestra.run.smithi161.stdout: 2024-04-10T22:28:43.665 INFO:teuthology.orchestra.run.smithi161.stdout:Complete! 2024-04-10T22:28:43.830 DEBUG:teuthology.orchestra.run.smithi161:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:28:44.048 INFO:teuthology.orchestra.run.smithi161.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-10T22:28:44.049 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-10T22:28:44.049 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-552.el8.x86_64 2024-04-10T22:28:44.049 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-10T22:28:44.049 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-10T22:28:44.049 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-04-10T22:28:44.049 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi161.front.sepia.ceph.com, path=None, version=distro) 2024-04-10T22:28:44.049 DEBUG:teuthology.orchestra.run.smithi161:> rpm -q kernel | sort -rV | head -n 1 2024-04-10T22:28:44.261 INFO:teuthology.orchestra.run.smithi161.stdout:kernel-4.18.0-552.el8.x86_64 2024-04-10T22:28:44.262 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-552.el8.x86_64 2024-04-10T22:28:44.262 DEBUG:teuthology.orchestra.run.smithi161:> sudo rpm -qi grub2-tools 2024-04-10T22:28:44.296 INFO:teuthology.orchestra.run.smithi161.stdout:Name : grub2-tools 2024-04-10T22:28:44.296 INFO:teuthology.orchestra.run.smithi161.stdout:Epoch : 1 2024-04-10T22:28:44.297 INFO:teuthology.orchestra.run.smithi161.stdout:Version : 2.02 2024-04-10T22:28:44.297 INFO:teuthology.orchestra.run.smithi161.stdout:Release : 129.el8 2024-04-10T22:28:44.297 INFO:teuthology.orchestra.run.smithi161.stdout:Architecture: x86_64 2024-04-10T22:28:44.297 INFO:teuthology.orchestra.run.smithi161.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2024-04-10T22:28:44.297 INFO:teuthology.orchestra.run.smithi161.stdout:Group : System Environment/Base 2024-04-10T22:28:44.297 INFO:teuthology.orchestra.run.smithi161.stdout:Size : 9265586 2024-04-10T22:28:44.297 INFO:teuthology.orchestra.run.smithi161.stdout:License : GPLv3+ 2024-04-10T22:28:44.297 INFO:teuthology.orchestra.run.smithi161.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2024-04-10T22:28:44.297 INFO:teuthology.orchestra.run.smithi161.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2024-04-10T22:28:44.297 INFO:teuthology.orchestra.run.smithi161.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2024-04-10T22:28:44.297 INFO:teuthology.orchestra.run.smithi161.stdout:Build Host : kbuilder.bsys.centos.org 2024-04-10T22:28:44.297 INFO:teuthology.orchestra.run.smithi161.stdout:Relocations : (not relocatable) 2024-04-10T22:28:44.297 INFO:teuthology.orchestra.run.smithi161.stdout:Packager : CentOS BuildSystem 2024-04-10T22:28:44.298 INFO:teuthology.orchestra.run.smithi161.stdout:Vendor : CentOS 2024-04-10T22:28:44.298 INFO:teuthology.orchestra.run.smithi161.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-10T22:28:44.298 INFO:teuthology.orchestra.run.smithi161.stdout:Summary : Support tools for GRUB. 2024-04-10T22:28:44.298 INFO:teuthology.orchestra.run.smithi161.stdout:Description : 2024-04-10T22:28:44.298 INFO:teuthology.orchestra.run.smithi161.stdout: 2024-04-10T22:28:44.298 INFO:teuthology.orchestra.run.smithi161.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-10T22:28:44.298 INFO:teuthology.orchestra.run.smithi161.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-10T22:28:44.298 INFO:teuthology.orchestra.run.smithi161.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-10T22:28:44.298 INFO:teuthology.orchestra.run.smithi161.stdout:hardware devices. 2024-04-10T22:28:44.298 INFO:teuthology.orchestra.run.smithi161.stdout: 2024-04-10T22:28:44.298 INFO:teuthology.orchestra.run.smithi161.stdout:This subpackage provides tools for support of all platforms. 2024-04-10T22:28:44.299 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-10T22:28:44.299 INFO:teuthology.task.kernel:Updating grub on smithi161 to boot 4.18.0-552.el8.x86_64 2024-04-10T22:28:44.300 DEBUG:teuthology.orchestra.run.smithi161:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-10T22:28:45.660 INFO:teuthology.orchestra.run.smithi161.stderr:Generating grub configuration file ... 2024-04-10T22:28:47.703 INFO:teuthology.orchestra.run.smithi161.stderr:done 2024-04-10T22:28:47.705 DEBUG:teuthology.orchestra.run.smithi161:> mktemp 2024-04-10T22:28:47.730 INFO:teuthology.orchestra.run.smithi161.stdout:/tmp/tmp.i6mz58dfJO 2024-04-10T22:28:47.730 DEBUG:teuthology.orchestra.run.smithi161:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.i6mz58dfJO 2024-04-10T22:28:47.804 DEBUG:teuthology.orchestra.run.smithi161:> sudo chmod 0666 /tmp/tmp.i6mz58dfJO 2024-04-10T22:28:47.951 DEBUG:teuthology.orchestra.remote:smithi161:/tmp/tmp.i6mz58dfJO is 6KB 2024-04-10T22:28:47.999 DEBUG:teuthology.orchestra.run.smithi161:> rm -fr /tmp/tmp.i6mz58dfJO 2024-04-10T22:28:48.017 DEBUG:teuthology.orchestra.run.smithi161:> sudo /bin/ls /boot/loader/entries || true 2024-04-10T22:28:48.089 INFO:teuthology.orchestra.run.smithi161.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2024-04-10T22:28:48.089 INFO:teuthology.orchestra.run.smithi161.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2024-04-10T22:28:48.089 INFO:teuthology.orchestra.run.smithi161.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2024-04-10T22:28:48.089 INFO:teuthology.orchestra.run.smithi161.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64.conf 2024-04-10T22:28:48.090 DEBUG:teuthology.orchestra.run.smithi161:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-552.el8.x86_64 2024-04-10T22:28:48.283 DEBUG:teuthology.orchestra.run.smithi161:> sudo shutdown -r now 2024-04-10T22:28:53.298 DEBUG:teuthology.orchestra.remote:timed out 2024-04-10T22:29:05.310 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi059.front.sepia.ceph.com' 2024-04-10T22:29:05.311 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi059.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:29:05.595 DEBUG:teuthology.orchestra.run.smithi059:> true 2024-04-10T22:29:05.881 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi059.front.sepia.ceph.com' 2024-04-10T22:29:05.881 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "4.18.0-552.el8.x86_64"... 2024-04-10T22:29:05.881 DEBUG:teuthology.orchestra.run.smithi059:> uname -r 2024-04-10T22:29:05.901 INFO:teuthology.orchestra.run.smithi059.stdout:4.18.0-552.el8.x86_64 2024-04-10T22:29:05.901 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-552.el8.x86_64 vs 4.18.0-552.el8.x86_64 2024-04-10T22:29:05.901 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-10T22:29:05.901 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-10T22:29:06.902 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-04-10T22:29:06.902 DEBUG:teuthology.orchestra.run.smithi059:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-10T22:29:06.976 INFO:teuthology.orchestra.run.smithi059.stdout:ttyS1 2024-04-10T22:29:06.989 DEBUG:teuthology.parallel:result is None 2024-04-10T22:29:18.286 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-04-10T22:29:18.287 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi161.front.sepia.ceph.com' 2024-04-10T22:29:18.287 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi161.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:29:36.670 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.161 2024-04-10T22:29:45.672 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi161.front.sepia.ceph.com' 2024-04-10T22:29:45.673 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi161.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:29:48.734 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.161 2024-04-10T22:30:00.735 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi161.front.sepia.ceph.com' 2024-04-10T22:30:00.736 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi161.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:31:00.739 DEBUG:teuthology.orchestra.remote:timed out 2024-04-10T22:31:15.741 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi161.front.sepia.ceph.com' 2024-04-10T22:31:15.743 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi161.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-10T22:31:16.123 DEBUG:teuthology.orchestra.run.smithi161:> true 2024-04-10T22:31:16.415 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi161.front.sepia.ceph.com' 2024-04-10T22:31:16.415 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "4.18.0-552.el8.x86_64"... 2024-04-10T22:31:16.415 DEBUG:teuthology.orchestra.run.smithi161:> uname -r 2024-04-10T22:31:16.433 INFO:teuthology.orchestra.run.smithi161.stdout:4.18.0-552.el8.x86_64 2024-04-10T22:31:16.434 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-552.el8.x86_64 vs 4.18.0-552.el8.x86_64 2024-04-10T22:31:16.434 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-10T22:31:16.434 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-10T22:31:17.434 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-04-10T22:31:17.435 DEBUG:teuthology.orchestra.run.smithi161:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-10T22:31:17.503 INFO:teuthology.orchestra.run.smithi161.stdout:ttyS1 2024-04-10T22:31:17.516 DEBUG:teuthology.parallel:result is None 2024-04-10T22:31:17.517 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-10T22:31:17.530 INFO:teuthology.task.internal:Creating test directory... 2024-04-10T22:31:17.530 DEBUG:teuthology.orchestra.run.smithi059:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-10T22:31:17.534 DEBUG:teuthology.orchestra.run.smithi161:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-10T22:31:17.555 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-10T22:31:17.563 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-10T22:31:17.570 INFO:teuthology.task.internal:Creating archive directory... 2024-04-10T22:31:17.571 DEBUG:teuthology.orchestra.run.smithi059:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-10T22:31:17.598 DEBUG:teuthology.orchestra.run.smithi161:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-10T22:31:17.634 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-10T22:31:17.640 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-10T22:31:17.641 DEBUG:teuthology.orchestra.run.smithi059:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-10T22:31:17.671 DEBUG:teuthology.orchestra.run.smithi161:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-10T22:31:17.714 INFO:teuthology.orchestra.run.smithi059.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:31:17.728 INFO:teuthology.orchestra.run.smithi059.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:31:17.729 INFO:teuthology.orchestra.run.smithi161.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:31:17.737 INFO:teuthology.orchestra.run.smithi161.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-10T22:31:17.738 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-10T22:31:17.746 INFO:teuthology.task.internal:Configuring sudo... 2024-04-10T22:31:17.746 DEBUG:teuthology.orchestra.run.smithi059:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-10T22:31:17.748 DEBUG:teuthology.orchestra.run.smithi161:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-10T22:31:17.778 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-10T22:31:17.787 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-10T22:31:17.787 DEBUG:teuthology.orchestra.run.smithi059:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-10T22:31:17.820 DEBUG:teuthology.orchestra.run.smithi161:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-10T22:31:17.841 DEBUG:teuthology.orchestra.run.smithi059:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:31:17.927 DEBUG:teuthology.orchestra.run.smithi059:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:31:17.995 DEBUG:teuthology.orchestra.run.smithi059:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:31:18.036 DEBUG:teuthology.orchestra.run.smithi059:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:31:18.087 DEBUG:teuthology.orchestra.run.smithi059:> set -ex 2024-04-10T22:31:18.087 DEBUG:teuthology.orchestra.run.smithi059:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-10T22:31:18.157 DEBUG:teuthology.orchestra.run.smithi161:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:31:18.203 DEBUG:teuthology.orchestra.run.smithi161:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-10T22:31:18.255 DEBUG:teuthology.orchestra.run.smithi161:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:31:18.337 DEBUG:teuthology.orchestra.run.smithi161:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-10T22:31:18.384 DEBUG:teuthology.orchestra.run.smithi161:> set -ex 2024-04-10T22:31:18.384 DEBUG:teuthology.orchestra.run.smithi161:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-10T22:31:18.451 DEBUG:teuthology.orchestra.run.smithi059:> sudo service rsyslog restart 2024-04-10T22:31:18.454 DEBUG:teuthology.orchestra.run.smithi161:> sudo service rsyslog restart 2024-04-10T22:31:18.523 INFO:teuthology.orchestra.run.smithi059.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:31:18.552 INFO:teuthology.orchestra.run.smithi161.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:31:19.002 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-10T22:31:19.011 INFO:teuthology.task.internal:Starting timer... 2024-04-10T22:31:19.011 INFO:teuthology.run_tasks:Running task pcp... 2024-04-10T22:31:19.030 INFO:teuthology.run_tasks:Running task selinux... 2024-04-10T22:31:19.039 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-04-10T22:31:19.039 DEBUG:teuthology.orchestra.run.smithi059:> sudo service auditd rotate 2024-04-10T22:31:19.107 INFO:teuthology.orchestra.run.smithi059.stdout:Rotating logs: 2024-04-10T22:31:19.109 DEBUG:teuthology.orchestra.run.smithi161:> sudo service auditd rotate 2024-04-10T22:31:19.205 INFO:teuthology.orchestra.run.smithi161.stdout:Rotating logs: 2024-04-10T22:31:19.206 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-10T22:31:19.207 DEBUG:teuthology.orchestra.run.smithi059:> /usr/sbin/getenforce 2024-04-10T22:31:19.231 INFO:teuthology.orchestra.run.smithi059.stdout:Permissive 2024-04-10T22:31:19.231 DEBUG:teuthology.orchestra.run.smithi161:> /usr/sbin/getenforce 2024-04-10T22:31:19.251 INFO:teuthology.orchestra.run.smithi161.stdout:Permissive 2024-04-10T22:31:19.251 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi059.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi161.front.sepia.ceph.com': 'permissive'} 2024-04-10T22:31:19.251 DEBUG:teuthology.orchestra.run.smithi059:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:31:19.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:31:19.302 DEBUG:teuthology.orchestra.run.smithi161:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:31:19.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:31:19.329 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-10T22:31:19.329 DEBUG:teuthology.orchestra.run.smithi059:> sudo /usr/sbin/setenforce permissive 2024-04-10T22:31:19.370 DEBUG:teuthology.orchestra.run.smithi161:> sudo /usr/sbin/setenforce permissive 2024-04-10T22:31:19.397 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-10T22:31:19.407 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-10T22:31:19.427 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-10T22:31:19.429 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi059.front.sepia.ceph.com,smithi161.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-10T22:45:37.670 INFO:teuthology.task.ansible:Archiving ansible failure log at: /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650168/ansible_failures.yaml 2024-04-10T22:45:37.672 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/__init__.py", line 123, in __enter__ self.begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 508, in begin super(CephLab, self).begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 342, in begin self.execute_playbook() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 374, in execute_playbook self._handle_failure(command, status) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 402, in _handle_failure raise AnsibleFailedError(failures) teuthology.exceptions.AnsibleFailedError: [',', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at'] 2024-04-10T22:45:37.898 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=ddc4047ba00f473ab5dbc502eeb24a73 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/__init__.py", line 123, in __enter__ self.begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 508, in begin super(CephLab, self).begin() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 342, in begin self.execute_playbook() File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 374, in execute_playbook self._handle_failure(command, status) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/task/ansible.py", line 402, in _handle_failure raise AnsibleFailedError(failures) teuthology.exceptions.AnsibleFailedError: [',', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at'] 2024-04-10T22:45:37.900 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-10T22:45:37.909 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-10T22:45:37.909 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-10T22:45:37.917 DEBUG:teuthology.orchestra.run.smithi059:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-10T22:45:38.064 DEBUG:teuthology.orchestra.run.smithi161:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-10T22:45:38.130 DEBUG:teuthology.orchestra.run.smithi059:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:45:38.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:45:38.163 DEBUG:teuthology.orchestra.run.smithi161:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-10T22:45:38.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:45:38.194 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-10T22:45:38.203 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-10T22:45:38.210 INFO:teuthology.task.internal:Duration was 859.198852 seconds 2024-04-10T22:45:38.210 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-10T22:45:38.218 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-10T22:45:38.218 DEBUG:teuthology.orchestra.run.smithi059:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-10T22:45:38.221 DEBUG:teuthology.orchestra.run.smithi161:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-10T22:45:38.271 INFO:teuthology.orchestra.run.smithi059.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:45:38.282 INFO:teuthology.orchestra.run.smithi161.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-10T22:45:38.502 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-10T22:45:38.502 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi059.front.sepia.ceph.com 2024-04-10T22:45:38.503 DEBUG:teuthology.orchestra.run.smithi059:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-10T22:45:38.529 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi161.front.sepia.ceph.com 2024-04-10T22:45:38.530 DEBUG:teuthology.orchestra.run.smithi161:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-10T22:45:38.553 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-10T22:45:38.553 DEBUG:teuthology.orchestra.run.smithi059:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-10T22:45:38.572 DEBUG:teuthology.orchestra.run.smithi161:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-10T22:45:38.633 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-10T22:45:38.634 DEBUG:teuthology.orchestra.run.smithi059:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-10T22:45:38.636 DEBUG:teuthology.orchestra.run.smithi161:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-10T22:45:38.735 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-10T22:45:38.746 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-10T22:45:38.747 DEBUG:teuthology.orchestra.run.smithi059:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-10T22:45:38.775 DEBUG:teuthology.orchestra.run.smithi161:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-10T22:45:38.802 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-10T22:45:38.812 DEBUG:teuthology.orchestra.run.smithi059:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-10T22:45:38.818 DEBUG:teuthology.orchestra.run.smithi161:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-10T22:45:38.843 INFO:teuthology.orchestra.run.smithi059.stdout:kernel.core_pattern = core 2024-04-10T22:45:38.868 INFO:teuthology.orchestra.run.smithi161.stdout:kernel.core_pattern = core 2024-04-10T22:45:38.895 DEBUG:teuthology.orchestra.run.smithi059:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-10T22:45:38.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:45:38.925 DEBUG:teuthology.orchestra.run.smithi161:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-10T22:45:38.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-10T22:45:38.953 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-10T22:45:38.963 INFO:teuthology.task.internal:Transferring archived files... 2024-04-10T22:45:38.965 DEBUG:teuthology.misc:Transferring archived files from smithi059:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650168/remote/smithi059 2024-04-10T22:45:38.966 DEBUG:teuthology.orchestra.run.smithi059:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-10T22:45:39.012 DEBUG:teuthology.misc:Transferring archived files from smithi161:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650168/remote/smithi161 2024-04-10T22:45:39.013 DEBUG:teuthology.orchestra.run.smithi161:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-10T22:45:39.060 INFO:teuthology.task.internal:Removing archive directory... 2024-04-10T22:45:39.061 DEBUG:teuthology.orchestra.run.smithi059:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-10T22:45:39.063 DEBUG:teuthology.orchestra.run.smithi161:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-10T22:45:39.113 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-10T22:45:39.124 INFO:teuthology.task.internal:Not uploading archives. 2024-04-10T22:45:39.124 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-10T22:45:39.132 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-10T22:45:39.132 DEBUG:teuthology.orchestra.run.smithi059:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-10T22:45:39.135 DEBUG:teuthology.orchestra.run.smithi161:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-10T22:45:39.150 INFO:teuthology.orchestra.run.smithi059.stdout: 393224 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 10 22:45 /home/ubuntu/cephtest 2024-04-10T22:45:39.170 INFO:teuthology.orchestra.run.smithi161.stdout: 393224 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 10 22:45 /home/ubuntu/cephtest 2024-04-10T22:45:39.171 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-10T22:45:39.181 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-10T22:45:39.198 INFO:teuthology.nuke:Checking targets against current locks 2024-04-10T22:45:39.256 DEBUG:teuthology.nuke:shortname: smithi059 2024-04-10T22:45:39.256 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T22:45:39.307 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi059.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650168', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:15:48.419823', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBP+sK/wAJLkoKgXQ1X30YFTzsciU1igIUuEVDD0kUsNtkRAfaunV5ewSK2d/BABRFGO2x5G+7aZLjo94mxQg5tQ='} 2024-04-10T22:45:39.309 DEBUG:teuthology.nuke:shortname: smithi161 2024-04-10T22:45:39.309 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-10T22:45:39.360 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi161.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-10_01:08:08-upgrade:pacific-x-reef-distro-default-smithi/7650168', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-10 22:15:48.421615', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBC3mPrcHfcLTa62/Rn/UXZQjZgaS1xhzzPZ8bedaU4ow+YLLPvZ7+BxF3dgrdrF8mJdTc5Xb31sOq2va0Dup3Xc='} 2024-04-10T22:45:39.436 INFO:teuthology.orchestra.console.smithi059:Power off 2024-04-10T22:45:39.437 DEBUG:teuthology.orchestra.console.smithi059:pexpect command: ipmitool -H smithi059.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-10T22:45:39.489 DEBUG:teuthology.orchestra.console.smithi059:power off output: Chassis Power Control: Down/Off 2024-04-10T22:45:39.490 DEBUG:teuthology.orchestra.console.smithi059:pexpect command: ipmitool -H smithi059.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:45:39.505 INFO:teuthology.orchestra.console.smithi161:Power off 2024-04-10T22:45:39.506 DEBUG:teuthology.orchestra.console.smithi161:pexpect command: ipmitool -H smithi161.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-10T22:45:39.543 DEBUG:teuthology.orchestra.console.smithi059:check power output: Chassis Power is on 2024-04-10T22:45:39.557 DEBUG:teuthology.orchestra.console.smithi161:power off output: Chassis Power Control: Down/Off 2024-04-10T22:45:39.557 DEBUG:teuthology.orchestra.console.smithi161:pexpect command: ipmitool -H smithi161.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:45:39.608 DEBUG:teuthology.orchestra.console.smithi161:check power output: Chassis Power is on 2024-04-10T22:45:43.545 DEBUG:teuthology.orchestra.console.smithi059:pexpect command: ipmitool -H smithi059.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:45:43.609 DEBUG:teuthology.orchestra.console.smithi161:pexpect command: ipmitool -H smithi161.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:45:43.655 DEBUG:teuthology.orchestra.console.smithi059:check power output: Chassis Power is on 2024-04-10T22:45:43.719 DEBUG:teuthology.orchestra.console.smithi161:check power output: Chassis Power is on 2024-04-10T22:45:47.658 DEBUG:teuthology.orchestra.console.smithi059:pexpect command: ipmitool -H smithi059.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:45:47.720 DEBUG:teuthology.orchestra.console.smithi161:pexpect command: ipmitool -H smithi161.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:45:47.769 DEBUG:teuthology.orchestra.console.smithi059:check power output: Chassis Power is on 2024-04-10T22:45:47.832 DEBUG:teuthology.orchestra.console.smithi161:check power output: Chassis Power is off 2024-04-10T22:45:47.932 INFO:teuthology.orchestra.console.smithi161:Power off completed 2024-04-10T22:45:51.770 DEBUG:teuthology.orchestra.console.smithi059:pexpect command: ipmitool -H smithi059.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-10T22:45:51.883 DEBUG:teuthology.orchestra.console.smithi059:check power output: Chassis Power is off 2024-04-10T22:45:51.983 INFO:teuthology.orchestra.console.smithi059:Power off completed 2024-04-10T22:45:52.090 INFO:teuthology.run:Summary data: description: upgrade:pacific-x/stress-split/{0-distro/centos_8.stream_container_tools 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} duration: 859.1988523006439 failure_reason: '['','', ''Bugzilla. In case of problems, contact the owner of this repository.'', ''Enabling a Copr repository. Please note that this repository is not part'', ''Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable'', ''Please do not file bug reports about these packages in Fedora'', ''The Fedora Project does not exercise any power over the contents of'', ''and packages are not held to any quality or security level.'', ''of the main distribution, and quality may vary.'', ''this repository beyond the rules outlined in the Copr FAQ at'']' owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=ddc4047ba00f473ab5dbc502eeb24a73 status: dead success: false 2024-04-10T22:45:52.090 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-10T22:45:52.198 INFO:teuthology.run:DEAD