Found 100 jobs Found 5 distinct failure reasons Timeout 3h running clone.client.0/qa/workunits/rbd/concurrent.sh 1 jobs: ['5717721'] suites: ['clusters/fixed-3', 'conf', 'install/ceph', 'krbd/rbd-nomount/{bluestore-bitmap', 'msgr-failures/many', 'tasks/rbd_concurrent}'] Failure: "2020-12-25T17:29:58.091105+0000 mon.b (mon.0) 186 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN)" in cluster log 1 jobs: ['5717666'] suites: ['clusters/3-node', 'conf', 'features/object-map', 'krbd/fsx/{ceph/ceph', 'objectstore/bluestore-bitmap', 'randomized-striping-on}', 'striping/fancy/{msgr-failures/few', 'tasks/fsx-3-client}'] Failure: Command failed on smithi069 with status 4: "/usr/bin/sudo TESTDIR=/home/ubuntu/cephtest adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage /bin/bash /home/ubuntu/cephtest/run_xfstests.sh -c 1 -f ext4 -t /dev/rbd0 -s /dev/rbd1 -x /tmp/exclude600trktq -r -- '-g auto -g blockdev -x clone'" 2 jobs: ['5717609', '5717692'] suites intersection: ['conf', 'krbd/singleton/{bluestore-bitmap', 'tasks/rbd_xfstests}'] suites union: ['conf', 'krbd/singleton/{bluestore-bitmap', 'msgr-failures/few', 'msgr-failures/many', 'tasks/rbd_xfstests}'] Failure: "2020-12-25T17:21:11.869450+0000 osd.0 (osd.0) 3 : cluster [WRN] slow request osd_op(client.4422.0:891648 3.c 3:36cd384a:::rbd_data.1146a53ee9b0.0000000000000002:head [write 0~0] snapc 22=[22] ondisk+write+known_if_redirected e55) initiated 2020-12-25T17:20:40.821036+0000 currently delayed" in cluster log 1 jobs: ['5717641'] suites: ['clusters/3-node', 'conf', 'features/no-object-map', 'krbd/fsx/{ceph/ceph', 'objectstore/bluestore-bitmap', 'randomized-striping-on}', 'striping/fancy/{msgr-failures/few', 'tasks/fsx-1-client}'] Failure: Command failed (workunit test rbd/krbd_stable_pages_required.sh) on smithi134 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=7a0bd03818725dacbf52f64ad4ec5e2d12f87bc2 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/krbd_stable_pages_required.sh' 1 jobs: ['5717615'] suites: ['ceph/ceph', 'clusters/fixed-1', 'conf', 'krbd/wac/sysfs/{bluestore-bitmap', 'tasks/stable_pages_required}']