Found 222 jobs Found 4 distinct failure reasons Failure: Command failed on smithi172 with status 1: "/usr/bin/sudo TESTDIR=/home/ubuntu/cephtest adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage /bin/bash /home/ubuntu/cephtest/run_xfstests.sh -c 1 -f ext4 -t /dev/rbd0 -s /dev/rbd1 -x /tmp/excluded5ke1_wl -r -- '-g auto -g blockdev -x clone'" 2 jobs: ['6902529', '6902641'] suites intersection: ['conf', 'krbd/singleton/{bluestore-bitmap', 'tasks/rbd_xfstests}'] suites union: ['conf', 'krbd/singleton/{bluestore-bitmap', 'ms_mode$/{legacy-rxbounce}', 'ms_mode$/{legacy}', 'msgr-failures/few', 'msgr-failures/many', 'tasks/rbd_xfstests}'] Failure: Failed to fetch package version from https://shaman.ceph.com/api/search/?status=ready&project=kernel&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=nightly_pre-single-major 1 jobs: ['6902531'] suites: ['clusters/separate-client', 'conf', 'filestore-xfs', 'kernels/pre-single-major', 'krbd/unmap/{ceph/ceph', 'tasks/unmap}'] Timeout 3h running clone.client.0/qa/workunits/rbd/concurrent.sh 1 jobs: ['6902587'] suites: ['clusters/fixed-3', 'conf', 'install/ceph', 'krbd/rbd-nomount/{bluestore-bitmap', 'ms_mode/crc$/{crc}', 'msgr-failures/many', 'tasks/rbd_concurrent}'] Failure: "2022-07-04T10:31:31.286256+0000 osd.1 (osd.1) 3 : cluster [WRN] slow request osd_op(client.4340.0:636829 2.4 2:26980a3d:::rbd_data.112b7af8d52a.000000000000049f:head [set-alloc-hint object_size 4194304 write_size 4194304,write 1724416~12288 in=12288b] snapc 0=[] RETRY=11 ondisk+retry+write+known_if_redirected e22) initiated 2022-07-04T10:31:00.655292+0000 currently waiting for sub ops" in cluster log 1 jobs: ['6902542'] suites: ['clusters/fixed-3', 'conf', 'krbd/rbd/{bluestore-bitmap', 'ms_mode/crc$/{crc}', 'msgr-failures/many', 'tasks/rbd_workunit_suites_ffsb}']