Found 220 jobs Found 4 distinct failure reasons Failure: "2021-12-14T11:10:51.173007+0000 mon.a (mon.0) 16 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN)" in cluster log 2 jobs: ['6562134', '6562252'] suites intersection: ['clusters/fixed-3', 'conf', 'msgr-failures/many'] suites union: ['clusters/fixed-3', 'conf', 'install/ceph', 'krbd/rbd-nomount/{bluestore-bitmap', 'krbd/rbd/{bluestore-bitmap', 'ms_mode/legacy', 'ms_mode/secure', 'msgr-failures/many', 'tasks/krbd_fallocate}', 'tasks/rbd_workunit_suites_ffsb}'] Failure: Command failed on smithi118 with status 128: 'cd /lib/firmware/updates && sudo git fetch origin && sudo git reset --hard origin/master' 1 jobs: ['6562227'] suites: ['clusters/fixed-3', 'conf', 'install/ceph', 'krbd/rbd-nomount/{bluestore-bitmap', 'ms_mode/crc', 'msgr-failures/many', 'tasks/rbd_image_read}'] Failure: Command failed on smithi174 with status 1: "/usr/bin/sudo TESTDIR=/home/ubuntu/cephtest adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage /bin/bash /home/ubuntu/cephtest/run_xfstests.sh -c 1 -f ext4 -t /dev/rbd0 -s /dev/rbd1 -x /tmp/exclude760wi32d -r -- '-g auto -g blockdev -x clone'" 2 jobs: ['6562093', '6562204'] suites intersection: ['conf', 'krbd/singleton/{bluestore-bitmap', 'tasks/rbd_xfstests}'] suites union: ['conf', 'krbd/singleton/{bluestore-bitmap', 'ms_mode$/{prefer-crc}', 'ms_mode$/{secure}', 'msgr-failures/few', 'msgr-failures/many', 'tasks/rbd_xfstests}'] Failure: Failed to fetch package version from https://shaman.ceph.com/api/search/?status=ready&project=kernel&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=nightly_pre-single-major 1 jobs: ['6562095'] suites: ['clusters/separate-client', 'conf', 'filestore-xfs', 'kernels/pre-single-major', 'krbd/unmap/{ceph/ceph', 'tasks/unmap}']