Found 98 jobs Found 5 distinct failure reasons Timeout 3h running clone.client.0/qa/workunits/rbd/concurrent.sh 2 jobs: ['5476473', '5476363'] suites intersection: ['clusters/fixed-3', 'conf', 'install/ceph', 'krbd/rbd-nomount/{bluestore-bitmap', 'tasks/rbd_concurrent}'] suites union: ['clusters/fixed-3', 'conf', 'install/ceph', 'krbd/rbd-nomount/{bluestore-bitmap', 'msgr-failures/few', 'msgr-failures/many', 'tasks/rbd_concurrent}'] Failure: "2020-10-04T15:16:38.199871+0000 mon.a (mon.0) 38 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY)" in cluster log 3 jobs: ['5476415', '5476397', '5476401'] suites intersection: ['conf'] suites union: ['clusters/3-node', 'clusters/fixed-3', 'conf', 'features/no-object-map', 'krbd/fsx/{ceph/ceph', 'krbd/rbd/{bluestore-bitmap', 'msgr-failures/many', 'objectstore/filestore-xfs', 'randomized-striping-on}', 'striping/fancy/{msgr-failures/few', 'tasks/fsx-1-client}', 'tasks/rbd_fio}', 'tasks/rbd_workunit_suites_iozone}'] Failure: Command failed on smithi052 with status 3: "/usr/bin/sudo TESTDIR=/home/ubuntu/cephtest adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage /bin/bash /home/ubuntu/cephtest/run_xfstests.sh -c 1 -f ext4 -t /dev/rbd0 -s /dev/rbd1 -x /tmp/excludel2a0gbh_ -r -- '-g auto -g blockdev -x clone'" 2 jobs: ['5476419', '5476255'] suites intersection: ['conf', 'krbd/singleton/{bluestore-bitmap', 'tasks/rbd_xfstests}'] suites union: ['conf', 'krbd/singleton/{bluestore-bitmap', 'msgr-failures/few', 'msgr-failures/many', 'tasks/rbd_xfstests}'] Failure: "2020-10-04T15:06:11.663563+0000 osd.7 (osd.7) 3 : cluster [WRN] slow request osd_op(client.4422.0:168772 4.1d 4:bda422e0:::rbd_data.11469f2fa818.0000000000000007:head [delete] snapc b=[] ondisk+write+known_if_redirected e307) initiated 2020-10-04T15:05:41.362742+0000 currently delayed" in cluster log 1 jobs: ['5476367'] suites: ['clusters/3-node', 'conf', 'features/object-map', 'krbd/fsx/{ceph/ceph', 'objectstore/bluestore-bitmap', 'randomized-striping-on}', 'striping/fancy/{msgr-failures/few', 'tasks/fsx-3-client}'] Failure: Command failed on smithi142 with status 151: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_librbd_fsx --cluster ceph --id 1 -d -W -R -p 100 -P /home/ubuntu/cephtest/archive -r 512 -w 512 -t 512 -h 512 -l 250000000 -S 0 -N 10000 -K -U pool_client.1 image_client.1' 1 jobs: ['5476457'] suites: ['clusters/3-node', 'conf', 'features/no-object-map', 'krbd/fsx/{ceph/ceph', 'objectstore/filestore-xfs', 'randomized-striping-off}', 'striping/default/{msgr-failures/many', 'tasks/fsx-3-client}']