Found 101 jobs Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Found 6 distinct failure reasons Failure: "2024-03-20T09:41:05.330307+0000 mon.smithi005 (mon.0) 118 : cluster [WRN] Health check failed: failed to probe daemons or devices (CEPHADM_REFRESH_FAILED)" in cluster log 73 jobs suites intersection: [] suites union: ['0-nvme-loop', '1-start', '2-ops/repave-all}', '2-ops/rm-zap-add}', '2-ops/rm-zap-flag}', '2-ops/rm-zap-wait}', '2-ops/rmdir-reactivate}', '2-services/basic', '2-services/client-keyring', '2-services/iscsi', '2-services/jaeger', '2-services/mirror', '2-services/nfs', '2-services/nfs-haproxy-proto', '2-services/nfs-ingress', '2-services/nfs-ingress-rgw-bucket', '2-services/nfs-ingress-rgw-user', '2-services/nfs-ingress2', '2-services/nfs-keepalive-only', '2-services/nfs2', '2-services/nvmeof', '2-services/rgw', '2-services/rgw-ingress', '2-thrash', '3-final}', '3-tasks/rados_api_tests', '3-tasks/radosbench', '3-tasks/small-objects', '3-tasks/snaps-few-objects', 'agent/off', 'agent/on', 'base/install', 'cluster/{fixed-3', 'conf/{client', 'conf/{disable-pool-app}', 'fixed-2', 'mds', 'mode/packaged', 'mode/root', 'mon', 'mon_election/classic', 'mon_election/connectivity', 'msgr/async', 'msgr/async-v1only', 'msgr/async-v2only', 'openstack}', 'orch:cephadm/nfs/{cluster/{1-node}', 'orch:cephadm/no-agent-workunits/{0-distro/centos_9.stream', 'orch:cephadm/no-agent-workunits/{0-distro/centos_9.stream_runc', 'orch:cephadm/no-agent-workunits/{0-distro/ubuntu_22.04', 'orch:cephadm/osds/{0-distro/centos_9.stream', 'orch:cephadm/osds/{0-distro/centos_9.stream_runc', 'orch:cephadm/osds/{0-distro/ubuntu_22.04', 'orch:cephadm/rbd_iscsi/{0-single-container-host', 'orch:cephadm/smoke-roleless/{0-distro/centos_9.stream', 'orch:cephadm/smoke-roleless/{0-distro/centos_9.stream_runc', 'orch:cephadm/smoke-roleless/{0-distro/ubuntu_22.04', 'orch:cephadm/smoke-singlehost/{0-random-distro$/{centos_9.stream_runc}', 'orch:cephadm/smoke-singlehost/{0-random-distro$/{ubuntu_22.04}', 'orch:cephadm/smoke-small/{0-distro/centos_9.stream_runc', 'orch:cephadm/smoke/{0-distro/centos_9.stream_runc', 'orch:cephadm/thrash/{0-distro/centos_9.stream', 'orch:cephadm/thrash/{0-distro/centos_9.stream_runc', 'orch:cephadm/thrash/{0-distro/ubuntu_22.04', 'orch:cephadm/with-work/{0-distro/centos_9.stream', 'orch:cephadm/with-work/{0-distro/centos_9.stream_runc', 'orch:cephadm/with-work/{0-distro/ubuntu_22.04', 'orch:cephadm/workunits/{0-distro/centos_9.stream', 'orch:cephadm/workunits/{0-distro/centos_9.stream_runc', 'orch:cephadm/workunits/{0-distro/ubuntu_22.04', 'osd}', 'overrides/ignorelist_health', 'root}', 'start', 'start}', 'supported-random-distros$/{ubuntu_latest}', 'task/test_ca_signed_key}', 'task/test_cephadm_timeout}', 'task/test_host_drain}', 'task/test_iscsi_container/{centos_9.stream', 'task/test_monitoring_stack_basic}', 'task/test_orch_cli_mon}', 'task/test_orch_cli}', 'task/test_rgw_multisite}', 'task/test_set_mon_crush_locations}', 'tasks/nfs}', 'tasks/rados_api_tests}', 'tasks/rados_python}', 'tasks/rotate-keys}', 'test_iscsi_container}}', 'workloads/cephadm_iscsi}'] Failure: Command failed on smithi043 with status 1: 'sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15fd193e-e6a0-11ee-95c9-87774f69a715 -e sha1=98d972ce3416f5a9a175c9ef4cfd79e8a3c3acc2 -- bash -c \'ceph versions | jq -e \'"\'"\'.osd | length == 2\'"\'"\'\'' 13 jobs: ['7612037', '7612021', '7612007', '7611973', '7611993', '7611989', '7612045', '7611974', '7612036', '7611983', '7612070', '7612056', '7612060'] suites intersection: [] suites union: ['0-nvme-loop', '1-bootstrap/17.2.0', '1-ranks/1', '1-ranks/2', '1-start', '1-volume/{0-create', '2-allow_standby_replay/no', '2-allow_standby_replay/yes', '2-client/fuse', '2-client/kclient', '2-nfs', '2-repo_digest/defaut', '3-inline/no', '3-inline/yes', '3-upgrade-mgr-staggered', '3-upgrade-with-workload', '3-upgrade/staggered', '4-config-upgrade/{fail_fs}', '4-final}', '4-verify}', '4-wait', '5-upgrade-ls', '5-upgrade-with-workload', '6-verify}}', 'agent/off', 'agent/on', 'centos_9.stream', 'conf/{client', 'fail_fs/no', 'fail_fs/yes', 'fixed-2', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon_election/classic', 'mon_election/classic}', 'mon_election/connectivity', 'orch:cephadm/mds_upgrade_sequence/{bluestore-bitmap', 'orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream', 'orch:cephadm/smoke-small/{0-distro/centos_9.stream_runc', 'orch:cephadm/smoke/{0-distro/ubuntu_22.04', 'orch:cephadm/upgrade/{1-start-distro/1-start-centos_9.stream', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'start}', 'syntax}', 'tasks/{0-from/quincy', 'tasks/{0-from/reef/{reef}', 'tasks/{0-from/reef/{v18.2.1}'] Failure: Command failed (workunit test cephadm/test_cephadm.sh) on smithi195 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=98d972ce3416f5a9a175c9ef4cfd79e8a3c3acc2 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephadm/test_cephadm.sh' 1 jobs: ['7612024'] suites: ['agent/on', 'mon_election/connectivity', 'orch:cephadm/workunits/{0-distro/ubuntu_22.04', 'task/test_cephadm}'] Failure: Command failed on smithi003 with status 1: 'sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:98d972ce3416f5a9a175c9ef4cfd79e8a3c3acc2 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 39ac65e0-e69d-11ee-95c9-87774f69a715 -- ceph-volume lvm zap /dev/nvme4n1' 6 jobs: ['7612009', '7612053', '7612016', '7611972', '7611992', '7612042'] suites intersection: ['agent/on'] suites union: ['0-nvme-loop', 'agent/on', 'fixed-2', 'mon_election/classic', 'mon_election/connectivity', 'orch:cephadm/smoke-small/{0-distro/centos_9.stream_runc', 'orch:cephadm/smoke/{0-distro/centos_9.stream', 'orch:cephadm/workunits/{0-distro/centos_9.stream', 'orch:cephadm/workunits/{0-distro/centos_9.stream_runc', 'orch:cephadm/workunits/{0-distro/ubuntu_22.04', 'start}', 'task/test_extra_daemon_features}', 'task/test_host_drain}', 'task/test_iscsi_container/{centos_9.stream', 'task/test_monitoring_stack_basic}', 'test_iscsi_container}}'] Dead: b'2024-03-20T21:29:04.958 DEBUG:teuthology.exit:Finished running handlers' 3 jobs: ['7612010', '7612059', '7612064'] suites intersection: [] suites union: ['0-nvme-loop', '1-start', '2-repo_digest/repo_digest', '2-services/nfs', '3-final}', '3-upgrade/simple', '4-wait', '5-upgrade-ls', 'agent/off', 'agent/on', 'mon_election/connectivity}', 'orch:cephadm/smoke-roleless/{0-distro/centos_9.stream_runc', 'orch:cephadm/upgrade/{1-start-distro/1-start-ubuntu_20.04'] Failure: Command failed on smithi165 with status 1: 'sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0a043bfe-e69f-11ee-95c9-87774f69a715 -e sha1=98d972ce3416f5a9a175c9ef4cfd79e8a3c3acc2 -- bash -c \'ceph versions | jq -e \'"\'"\'.overall | length == 1\'"\'"\'\'' 1 jobs: ['7612031'] suites: ['1-ranks/2', '1-volume/{0-create', '2-allow_standby_replay/no', '2-client/kclient', '3-inline/no', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_9.stream', 'conf/{client', 'fail_fs/no', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'orch:cephadm/mds_upgrade_sequence/{bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}', 'tasks/{0-from/quincy']