Found 172 jobs Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Missing teuthology log /home/teuthworker/archive/teuthology-2024-04-08_21:08:03-orch-squid-distro-default-smithi/7646341/teuthology.log Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Found 12 distinct failure reasons Failure: Command failed (workunit test rados/test.sh) on smithi050 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=4870d5a890743a871e99c190e2baea5ccf0df349 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rados/test.sh' 1 jobs: ['7646223'] suites: ['1-start', '2-thrash', '3-tasks/rados_api_tests', 'fixed-2', 'msgr/async-v1only', 'orch/cephadm/thrash/{0-distro/ubuntu_22.04', 'root}'] Failure: "2024-04-08T23:12:18.586458+0000 mon.a (mon.0) 483 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY)" in cluster log 31 jobs suites intersection: [] suites union: ['0-nvme-loop', '1-bootstrap/17.2.0', '1-start', '2-nfs', '2-repo_digest/defaut', '2-repo_digest/repo_digest', '2-thrash', '3-tasks/rados_api_tests', '3-tasks/radosbench', '3-tasks/small-objects', '3-tasks/snaps-few-objects', '3-upgrade-with-workload', '3-upgrade/simple', '3-upgrade/staggered', '4-final}', '4-wait', '5-upgrade-ls', 'agent/off', 'agent/on', 'base/install', 'cluster/{fixed-3', 'conf/{disable-pool-app}', 'fixed-2', 'mode/packaged', 'mode/root', 'mon_election/classic', 'mon_election/classic}', 'mon_election/connectivity', 'mon_election/connectivity}', 'msgr/async', 'msgr/async-v1only', 'msgr/async-v2only', 'openstack}', 'orch/cephadm/mgr-nfs-upgrade/{0-centos_9.stream', 'orch/cephadm/rbd_iscsi/{0-single-container-host', 'orch/cephadm/smoke-small/{0-distro/centos_9.stream_runc', 'orch/cephadm/smoke/{0-distro/centos_9.stream', 'orch/cephadm/thrash/{0-distro/centos_9.stream', 'orch/cephadm/thrash/{0-distro/centos_9.stream_runc', 'orch/cephadm/thrash/{0-distro/ubuntu_22.04', 'orch/cephadm/upgrade/{1-start-distro/1-start-centos_9.stream', 'orch/cephadm/upgrade/{1-start-distro/1-start-ubuntu_22.04', 'orch/cephadm/with-work/{0-distro/centos_9.stream', 'orch/cephadm/with-work/{0-distro/centos_9.stream_runc', 'orch/cephadm/with-work/{0-distro/ubuntu_22.04', 'orch/cephadm/workunits/{0-distro/ubuntu_22.04', 'root}', 'start', 'start}', 'task/test_host_drain}', 'tasks/rados_api_tests}', 'tasks/rados_python}', 'workloads/cephadm_iscsi}'] Failure: Command failed on smithi096 with status 100: "sudo apt update && sudo apt install -y apt-transport-https ca-certificates curl && sudo curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg && echo 'deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main' | sudo tee /etc/apt/sources.list.d/kubernetes.list && sudo apt update && sudo apt install -y kubelet kubeadm kubectl bridge-utils" 4 jobs: ['7646278', '7646231', '7646189', '7646319'] suites intersection: ['0-kubeadm', '0-nvme-loop', '1-rook', 'k8s/1.21', 'orch/rook/smoke/{0-distro/ubuntu_22.04'] suites union: ['0-kubeadm', '0-nvme-loop', '1-rook', '2-workload/none', '2-workload/radosbench', 'cluster/1-node', 'cluster/3-node', 'k8s/1.21', 'net/calico', 'net/flannel', 'net/host', 'orch/rook/smoke/{0-distro/ubuntu_22.04', 'rook/1.7.2}', 'rook/master}'] Dead: b'2024-04-09T10:23:31.770 DEBUG:teuthology.exit:Finished running handlers' 2 jobs: ['7646182', '7646190'] suites intersection: [] suites union: ['0-nvme-loop', '1-ranks/2', '1-start', '1-volume/{0-create', '2-allow_standby_replay/yes', '2-client/kclient', '2-services/client-keyring', '3-final}', '3-inline/yes', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_9.stream', 'conf/{client', 'fail_fs/no', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'orch/cephadm/mds_upgrade_sequence/{bluestore-bitmap', 'orch/cephadm/smoke-roleless/{0-distro/centos_9.stream_runc', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}', 'tasks/{0-from/quincy'] Failure: "2024-04-08T23:20:00.000151+0000 mon.a (mon.0) 1193 : cluster 3 [WRN] PG_BACKFILL_FULL: Low space hindering backfill (add storage if this doesn't resolve itself): 3 pgs backfill_toofull" in cluster log 1 jobs: ['7646237'] suites: ['1-start', '2-thrash', '3-tasks/radosbench', 'fixed-2', 'msgr/async-v2only', 'orch/cephadm/thrash/{0-distro/centos_9.stream', 'root}'] Dead: None 1 jobs: ['7646341'] Failure: Command failed on smithi121 with status 1: "sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a0fdd25c-f70e-11ee-b64a-cb9ed24678a4 -e sha1=b6f3e469c8285b60ad2826d6f524be2e6014ed05 -- bash -c 'ceph orch ps'" 3 jobs: ['7646282', '7646226', '7646302'] suites intersection: [] suites union: ['1-ranks/1', '1-volume/{0-create', '2-allow_standby_replay/no', '2-client/kclient', '2-repo_digest/repo_digest', '3-inline/yes', '3-upgrade-mgr-staggered', '3-upgrade/staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '4-wait', '5-upgrade-ls', '5-upgrade-with-workload', '6-verify}}', 'agent/on', 'centos_9.stream', 'conf/{client', 'fail_fs/no', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'mon_election/classic}', 'mon_election/connectivity}', 'orch/cephadm/mds_upgrade_sequence/{bluestore-bitmap', 'orch/cephadm/upgrade/{1-start-distro/1-start-ubuntu_22.04', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}', 'tasks/{0-from/quincy'] Failure: reached maximum tries (51) after waiting for 300 seconds 6 jobs: ['7646308', '7646230', '7646261', '7646336', '7646169', '7646204'] suites intersection: ['1-volume/{0-create', '2-client/fuse', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_9.stream', 'conf/{client', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'orch/cephadm/mds_upgrade_sequence/{bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}'] suites union: ['1-ranks/1', '1-ranks/2', '1-volume/{0-create', '2-allow_standby_replay/no', '2-allow_standby_replay/yes', '2-client/fuse', '3-inline/no', '3-inline/yes', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_9.stream', 'conf/{client', 'fail_fs/no', 'fail_fs/yes', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'orch/cephadm/mds_upgrade_sequence/{bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}', 'tasks/{0-from/quincy', 'tasks/{0-from/reef/{reef}', 'tasks/{0-from/reef/{v18.2.1}'] Timeout 3h running clone.client.1/qa/workunits/suites/fsstress.sh 2 jobs: ['7646216', '7646245'] suites intersection: ['1-volume/{0-create', '2-client/kclient', '3-inline/yes', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_9.stream', 'conf/{client', 'fail_fs/no', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'orch/cephadm/mds_upgrade_sequence/{bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}', 'tasks/{0-from/quincy'] suites union: ['1-ranks/1', '1-ranks/2', '1-volume/{0-create', '2-allow_standby_replay/no', '2-allow_standby_replay/yes', '2-client/kclient', '3-inline/yes', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_9.stream', 'conf/{client', 'fail_fs/no', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'orch/cephadm/mds_upgrade_sequence/{bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}', 'tasks/{0-from/quincy'] Failure: Command failed (workunit test cephadm/test_cephadm.sh) on smithi018 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=4870d5a890743a871e99c190e2baea5ccf0df349 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephadm/test_cephadm.sh' 2 jobs: ['7646213', '7646288'] suites intersection: ['agent/on', 'task/test_cephadm}'] suites union: ['agent/on', 'mon_election/classic', 'mon_election/connectivity', 'orch/cephadm/workunits/{0-distro/centos_9.stream_runc', 'orch/cephadm/workunits/{0-distro/ubuntu_22.04', 'task/test_cephadm}'] Failure: Command failed (workunit test cephadm/test_repos.sh) on smithi156 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=4870d5a890743a871e99c190e2baea5ccf0df349 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephadm/test_repos.sh' 1 jobs: ['7646294'] suites: ['agent/off', 'mon_election/classic', 'orch/cephadm/workunits/{0-distro/ubuntu_22.04', 'task/test_cephadm_repos}'] Failure: Command failed on smithi008 with status 5: 'sudo systemctl stop ceph-3e16b038-f717-11ee-b64a-cb9ed24678a4@mon.a' 1 jobs: ['7646342'] suites: ['agent/on', 'mon_election/connectivity', 'orch/cephadm/workunits/{0-distro/ubuntu_22.04', 'task/test_ca_signed_key}']