Found 220 jobs Found 5 distinct failure reasons Failure: Command failed on smithi193 with status 151: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_librbd_fsx --cluster ceph --id 2 -d -W -R -p 100 -P /home/ubuntu/cephtest/archive -r 512 -w 512 -t 512 -h 512 -l 250000000 -S 0 -N 10000 -K -U pool_client.2 image_client.2' 1 jobs: ['6051297'] suites: ['clusters/3-node', 'conf', 'features/object-map', 'krbd/fsx/{ceph/ceph', 'ms_mode$/{crc}', 'objectstore/filestore-xfs', 'randomized-striping-off}', 'striping/default/{msgr-failures/many', 'tasks/fsx-3-client}'] Dead: b'2021-04-18T09:05:12.947 DEBUG:teuthology.exit:Finished running handlers' 1 jobs: ['6051253'] suites: ['clusters/separate-client', 'conf', 'filestore-xfs', 'kernels/pre-single-major', 'krbd/unmap/{ceph/ceph', 'tasks/unmap}'] Failure: "2021-04-17T23:55:50.043059+0000 osd.5 (osd.5) 3 : cluster [WRN] slow request osd_op(client.11418.0:341 2.3 2:c4c92e5a:::rbd_trash:head [call rbd.trash_remove in=31b] snapc 0=[] ondisk+write+known_if_redirected+full_try e20) initiated 2021-04-17T23:55:19.872294+0000 currently delayed" in cluster log 1 jobs: ['6051455'] suites: ['clusters/fixed-3', 'conf', 'install/ceph', 'krbd/rbd-nomount/{bluestore-bitmap', 'ms_mode/secure', 'msgr-failures/many', 'tasks/rbd_concurrent}'] Failure: "2021-04-17T22:03:24.718644+0000 osd.7 (osd.7) 3 : cluster [WRN] slow request osd_op(client.7294.0:276 2.5 2:ac87f0a8:::rbd_data.1c5161a77336.00000000000000cd:head [delete] snapc 0=[] ondisk+write+known_if_redirected e20) initiated 2021-04-17T22:02:54.700531+0000 currently started" in cluster log 1 jobs: ['6051309'] suites: ['clusters/fixed-3', 'conf', 'install/ceph', 'krbd/rbd-nomount/{bluestore-bitmap', 'ms_mode/crc', 'msgr-failures/many', 'tasks/rbd_concurrent}'] Failure: "2021-04-17T22:39:18.240619+0000 mon.a (mon.0) 132 : cluster [WRN] Health check failed: 0 slow ops, oldest one blocked for 30 sec, osd.1 has slow ops (SLOW_OPS)" in cluster log 1 jobs: ['6051360'] suites: ['clusters/fixed-3', 'conf', 'krbd/rbd/{bluestore-bitmap', 'ms_mode/crc', 'msgr-failures/many', 'tasks/rbd_fio}']