Found 220 jobs Found 5 distinct failure reasons Failure: "2021-08-20T07:12:25.488292+0000 osd.5 (osd.5) 3 : cluster [WRN] slow request osd_op(client.4401.0:128933 2.3 2:dcb08865:::rbd_data.112b56bbf4d9.00000000000000c1:head [set-alloc-hint object_size 4194304 write_size 4194304,write 4124672~16384 in=16384b] snapc 0=[] ondisk+write+known_if_redirected e20) initiated 2021-08-20T07:11:55.471565+0000 currently waiting for sub ops" in cluster log 2 jobs: ['6348581', '6348524'] suites intersection: ['clusters/fixed-3', 'conf', 'krbd/rbd/{bluestore-bitmap', 'ms_mode/secure', 'msgr-failures/many'] suites union: ['clusters/fixed-3', 'conf', 'krbd/rbd/{bluestore-bitmap', 'ms_mode/secure', 'msgr-failures/many', 'tasks/rbd_workunit_suites_dbench}', 'tasks/rbd_workunit_suites_fsx}'] Failure: "2021-08-20T06:24:04.734430+0000 mon.a (mon.0) 135 : cluster [WRN] Health check failed: 0 slow ops, oldest one blocked for 33 sec, osd.5 has slow ops (SLOW_OPS)" in cluster log 1 jobs: ['6348522'] suites: ['clusters/fixed-3', 'conf', 'install/ceph', 'krbd/rbd-nomount/{bluestore-bitmap', 'ms_mode/crc', 'msgr-failures/many', 'tasks/rbd_concurrent}'] Failure: Command failed on smithi074 with status 151: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_librbd_fsx --cluster ceph --id 1 -d -W -R -p 100 -P /home/ubuntu/cephtest/archive -r 512 -w 512 -t 512 -h 512 -l 250000000 -S 0 -N 10000 -K -U pool_client.1 image_client.1' 2 jobs: ['6348564', '6348592'] suites intersection: ['clusters/3-node', 'conf', 'features/object-map', 'krbd/fsx/{ceph/ceph', 'objectstore/bluestore-bitmap', 'randomized-striping-off}', 'striping/default/{msgr-failures/many'] suites union: ['clusters/3-node', 'conf', 'features/object-map', 'krbd/fsx/{ceph/ceph', 'ms_mode$/{legacy}', 'ms_mode$/{prefer-crc}', 'objectstore/bluestore-bitmap', 'randomized-striping-off}', 'striping/default/{msgr-failures/many', 'tasks/fsx-1-client}', 'tasks/fsx-3-client}'] Failure: Failed to fetch package version from https://shaman.ceph.com/api/search/?status=ready&project=kernel&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=nightly_pre-single-major 1 jobs: ['6348466'] suites: ['clusters/separate-client', 'conf', 'filestore-xfs', 'kernels/pre-single-major', 'krbd/unmap/{ceph/ceph', 'tasks/unmap}'] Failure: Command failed on smithi158 with status 1: "/usr/bin/sudo TESTDIR=/home/ubuntu/cephtest adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage /bin/bash /home/ubuntu/cephtest/run_xfstests.sh -c 1 -f ext4 -t /dev/rbd0 -s /dev/rbd1 -x /tmp/exclude8ld2ac1s -r -- '-g auto -g blockdev -x clone'" 2 jobs: ['6348575', '6348464'] suites intersection: ['conf', 'krbd/singleton/{bluestore-bitmap', 'tasks/rbd_xfstests}'] suites union: ['conf', 'krbd/singleton/{bluestore-bitmap', 'ms_mode$/{legacy}', 'ms_mode$/{secure}', 'msgr-failures/few', 'msgr-failures/many', 'tasks/rbd_xfstests}']