Found 96 jobs Found 6 distinct failure reasons Failure: "2020-06-29T05:32:28.599900+0000 mon.a (mon.0) 126 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY)" in cluster log 16 jobs: ['5187768', '5187760', '5187775', '5187787', '5187785', '5187807', '5187764', '5187781', '5187736', '5187744', '5187806', '5187802', '5187753', '5187808', '5187793', '5187746'] suites intersection: ['conf'] suites union: ['clusters/3-node', 'clusters/fixed-3', 'conf', 'features/no-object-map', 'features/object-map', 'install/ceph', 'krbd/fsx/{ceph/ceph', 'krbd/rbd-nomount/{bluestore-bitmap', 'krbd/rbd/{bluestore-bitmap', 'msgr-failures/few', 'msgr-failures/many', 'objectstore/bluestore-bitmap', 'objectstore/filestore-xfs', 'randomized-striping-off}', 'randomized-striping-on}', 'striping/default/{msgr-failures/few', 'striping/default/{msgr-failures/many', 'striping/fancy/{msgr-failures/few', 'tasks/fsx-1-client}', 'tasks/fsx-3-client}', 'tasks/krbd_data_pool}', 'tasks/krbd_fallocate}', 'tasks/krbd_latest_osdmap_on_map}', 'tasks/krbd_udev_enumerate}', 'tasks/krbd_udev_netlink_enobufs}', 'tasks/rbd_concurrent}', 'tasks/rbd_workunit_suites_ffsb}', 'tasks/rbd_workunit_suites_fsstress}', 'tasks/rbd_workunit_trivial_sync}'] Failure: "2020-06-29T05:13:20.053310+0000 osd.3 (osd.3) 3 : cluster [WRN] slow request osd_op(client.4465.0:13 3.1e 3.66ccf43e (undecoded) ondisk+write+known_if_redirected e22) initiated 2020-06-29T05:12:49.145594+0000 currently delayed" in cluster log 4 jobs: ['5187751', '5187801', '5187779', '5187811'] suites intersection: ['clusters/3-node', 'conf', 'krbd/fsx/{ceph/ceph'] suites union: ['clusters/3-node', 'conf', 'features/no-object-map', 'features/object-map', 'krbd/fsx/{ceph/ceph', 'objectstore/bluestore-bitmap', 'objectstore/filestore-xfs', 'randomized-striping-off}', 'randomized-striping-on}', 'striping/default/{msgr-failures/few', 'striping/fancy/{msgr-failures/few', 'tasks/fsx-1-client}', 'tasks/fsx-3-client}'] Failure: Command failed on smithi013 with status 1: 'CEPH_REF=master CEPH_ID="0" PATH=$PATH:/usr/sbin adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage /home/ubuntu/cephtest/virtualenv/bin/cram -v -- /home/ubuntu/cephtest/archive/cram.client.0/*.t' 5 jobs: ['5187750', '5187797', '5187810', '5187821', '5187735'] suites intersection: ['ceph/ceph', 'clusters/fixed-1', 'conf', 'krbd/basic/{bluestore-bitmap'] suites union: ['ceph/ceph', 'clusters/fixed-1', 'conf', 'krbd/basic/{bluestore-bitmap', 'tasks/krbd_deep_flatten}', 'tasks/krbd_discard}', 'tasks/krbd_parent_overlap}', 'tasks/krbd_read_only}', 'tasks/krbd_whole_object_zeroout}'] Failure: Command failed on smithi164 with status 3: "/usr/bin/sudo TESTDIR=/home/ubuntu/cephtest adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage /bin/bash /home/ubuntu/cephtest/run_xfstests.sh -c 1 -f ext4 -t /dev/rbd0 -s /dev/rbd1 -x /tmp/exclude658pjljn -r -- '-g auto -g blockdev -x clone'" 2 jobs: ['5187788', '5187739'] suites intersection: ['conf', 'krbd/singleton/{bluestore-bitmap', 'tasks/rbd_xfstests}'] suites union: ['conf', 'krbd/singleton/{bluestore-bitmap', 'msgr-failures/few', 'msgr-failures/many', 'tasks/rbd_xfstests}'] Failure: Scrubbing terminated -- not all pgs were active and clean. 1 jobs: ['5187823'] suites: ['ceph/ceph', 'clusters/fixed-3', 'conf', 'krbd/thrash/{bluestore-bitmap', 'thrashers/upmap', 'thrashosds-health', 'workloads/rbd_fio}'] Timeout 3h running clone.client.0/qa/workunits/suites/ffsb.sh 1 jobs: ['5187776'] suites: ['ceph/ceph', 'clusters/fixed-3', 'conf', 'krbd/thrash/{bluestore-bitmap', 'thrashers/upmap', 'thrashosds-health', 'workloads/rbd_workunit_suites_ffsb}']