Found 221 jobs Found 4 distinct failure reasons Failure: Failed to fetch package version from https://shaman.ceph.com/api/search/?status=ready&project=kernel&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=nightly_pre-single-major 1 jobs: ['6726880'] suites: ['clusters/separate-client', 'conf', 'filestore-xfs', 'kernels/pre-single-major', 'krbd/unmap/{ceph/ceph', 'tasks/unmap}'] Failure: "2022-03-09T13:44:40.236093+0000 osd.4 (osd.4) 3 : cluster [WRN] slow request osd_op(client.4346.0:187454 2.6 2:7af403a4:::rbd_data.10f17df15357.0000000000001218:head [set-alloc-hint object_size 4194304 write_size 4194304,write 905216~4096 in=4096b] snapc 0=[] ondisk+write+known_if_redirected e22) initiated 2022-03-09T13:44:10.109867+0000 currently waiting for sub ops" in cluster log 2 jobs: ['6727037', '6726955'] suites intersection: ['clusters/fixed-3', 'conf', 'krbd/rbd/{bluestore-bitmap', 'ms_mode/secure', 'msgr-failures/many'] suites union: ['clusters/fixed-3', 'conf', 'krbd/rbd/{bluestore-bitmap', 'ms_mode/secure', 'msgr-failures/many', 'tasks/rbd_workunit_kernel_untar_build}', 'tasks/rbd_workunit_suites_ffsb}'] Failure: Command failed on smithi164 with status 2: "/usr/bin/sudo TESTDIR=/home/ubuntu/cephtest adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage /bin/bash /home/ubuntu/cephtest/run_xfstests.sh -c 1 -f ext4 -t /dev/rbd0 -s /dev/rbd1 -x /tmp/excludectlyye8f -r -- '-g auto -g blockdev -x clone'" 2 jobs: ['6726989', '6726878'] suites intersection: ['conf', 'krbd/singleton/{bluestore-bitmap', 'tasks/rbd_xfstests}'] suites union: ['conf', 'krbd/singleton/{bluestore-bitmap', 'ms_mode$/{crc}', 'ms_mode$/{legacy-rxbounce}', 'msgr-failures/few', 'msgr-failures/many', 'tasks/rbd_xfstests}'] Failure: "2022-03-09T10:38:52.872578+0000 mon.a (mon.0) 135 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN)" in cluster log 3 jobs: ['6726898', '6727068', '6727035'] suites intersection: ['conf'] suites union: ['clusters/3-node', 'clusters/fixed-3', 'conf', 'features/no-object-map', 'install/ceph', 'krbd/fsx/{ceph/ceph', 'krbd/rbd-nomount/{bluestore-bitmap', 'ms_mode$/{crc}', 'ms_mode/crc$/{crc-rxbounce}', 'ms_mode/secure', 'msgr-failures/many', 'objectstore/filestore-xfs', 'randomized-striping-off}', 'striping/default/{msgr-failures/many', 'tasks/fsx-3-client}', 'tasks/krbd_latest_osdmap_on_map}', 'tasks/krbd_udev_symlinks}']