Found 362 jobs Found 4 distinct failure reasons Failure: Command failed on smithi186 with status 22: 'kubectl -n rook-ceph exec rook-ceph-tools-78cdfd976c-dhmph -- ceph orch ps' 3 jobs: ['6180617', '6180391', '6180276'] suites intersection: ['0-kubeadm', '1-rook', '3-final', 'k8s/1.21', 'net/calico'] suites union: ['0-kubeadm', '1-rook', '2-workload/none', '2-workload/radosbench', '3-final', 'cluster/1-node', 'cluster/3-node', 'k8s/1.21', 'net/calico', 'rados/rook/smoke/{0-distro/ubuntu_18.04', 'rados/rook/smoke/{0-distro/ubuntu_20.04', 'rook/1.6.2}', 'rook/master}'] Failure: "2021-06-20T04:45:53.847616+0000 mon.a (mon.0) 48 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN)" in cluster log 1 jobs: ['6180405'] suites: ['cluster/{12-osds', 'mon_election/connectivity', 'msgr-failures/few', 'objectstore/filestore-xfs', 'openstack}', 'rados', 'rados/thrash-erasure-code-big/{ceph', 'recovery-overrides/{more-async-partial-recovery}', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/careful', 'thrashosds-health', 'workloads/ec-rados-plugin=lrc-k=4-m=2-l=3}'] Failure: Command failed (workunit test rados/test.sh) on smithi112 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=41a8f5ee6f625249b78c7fa8421419a771bf4076 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rados/test.sh' 1 jobs: ['6180540'] suites: ['1-start', '2-thrash', '3-tasks/rados_api_tests', 'fixed-2', 'msgr/async', 'rados/cephadm/thrash/{0-distro/centos_8.2_kubic_stable', 'root}'] Failure: 'check osd count' reached maximum tries (90) after waiting for 900 seconds 1 jobs: ['6180507'] suites: ['0-kubeadm', '1-rook', '2-workload/none', '3-final', 'cluster/3-node', 'k8s/1.21', 'net/calico', 'rados/rook/smoke/{0-distro/ubuntu_18.04', 'rook/1.6.2}']