Found 220 jobs Found 5 distinct failure reasons Failure: "2021-11-12T00:02:13.738121+0000 mon.a (mon.0) 242 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN)" in cluster log 31 jobs suites intersection: ['conf'] suites union: ['clusters/3-node', 'clusters/fixed-3', 'conf', 'features/no-object-map', 'features/object-map', 'install/ceph', 'krbd/fsx/{ceph/ceph', 'krbd/rbd-nomount/{bluestore-bitmap', 'krbd/rbd/{bluestore-bitmap', 'ms_mode$/{crc}', 'ms_mode$/{prefer-crc}', 'ms_mode/crc', 'ms_mode/legacy', 'ms_mode/secure', 'msgr-failures/few', 'msgr-failures/many', 'objectstore/bluestore-bitmap', 'objectstore/filestore-xfs', 'randomized-striping-off}', 'randomized-striping-on}', 'striping/default/{msgr-failures/many', 'striping/fancy/{msgr-failures/few', 'tasks/fsx-1-client}', 'tasks/fsx-3-client}', 'tasks/krbd_data_pool}', 'tasks/krbd_huge_osdmap}', 'tasks/krbd_latest_osdmap_on_map}', 'tasks/krbd_namespaces}', 'tasks/krbd_udev_enumerate}', 'tasks/krbd_udev_netlink_enobufs}', 'tasks/krbd_udev_netns}', 'tasks/krbd_udev_symlinks}', 'tasks/rbd_huge_tickets}', 'tasks/rbd_image_read}', 'tasks/rbd_kernel}', 'tasks/rbd_map_snapshot_io}', 'tasks/rbd_simple_big}', 'tasks/rbd_workunit_kernel_untar_build}', 'tasks/rbd_workunit_suites_ffsb}', 'tasks/rbd_workunit_suites_fsstress_ext4}', 'tasks/rbd_workunit_suites_fsstress}', 'tasks/rbd_workunit_suites_fsx}', 'tasks/rbd_workunit_suites_iozone}', 'tasks/rbd_workunit_trivial_sync}'] Failure: Command failed on gibba037 with status 1: "/usr/bin/sudo TESTDIR=/home/ubuntu/cephtest adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage /bin/bash /home/ubuntu/cephtest/run_xfstests.sh -c 1 -f ext4 -t /dev/rbd0 -s /dev/rbd1 -x /tmp/excludefx34980b -r -- '-g auto -g blockdev -x clone'" 2 jobs: ['6486199', '6486310'] suites intersection: ['conf', 'krbd/singleton/{bluestore-bitmap', 'ms_mode$/{crc}', 'tasks/rbd_xfstests}'] suites union: ['conf', 'krbd/singleton/{bluestore-bitmap', 'ms_mode$/{crc}', 'msgr-failures/few', 'msgr-failures/many', 'tasks/rbd_xfstests}'] Failure: reached maximum tries (90) after waiting for 540 seconds 3 jobs: ['6486356', '6486245', '6486200'] suites intersection: ['clusters/3-node', 'conf', 'krbd/fsx/{ceph/ceph', 'objectstore/filestore-xfs', 'randomized-striping-off}', 'striping/default/{msgr-failures/many', 'tasks/fsx-3-client}'] suites union: ['clusters/3-node', 'conf', 'features/no-object-map', 'features/object-map', 'krbd/fsx/{ceph/ceph', 'ms_mode$/{crc}', 'ms_mode$/{prefer-crc}', 'objectstore/filestore-xfs', 'randomized-striping-off}', 'striping/default/{msgr-failures/many', 'tasks/fsx-3-client}'] Failure: SSH connection to gibba029 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0' 2 jobs: ['6486257', '6486403'] suites intersection: ['clusters/fixed-3', 'conf', 'install/ceph', 'krbd/rbd-nomount/{bluestore-bitmap', 'msgr-failures/many', 'tasks/rbd_concurrent}'] suites union: ['clusters/fixed-3', 'conf', 'install/ceph', 'krbd/rbd-nomount/{bluestore-bitmap', 'ms_mode/crc', 'ms_mode/secure', 'msgr-failures/many', 'tasks/rbd_concurrent}'] Failure: Failed to fetch package version from https://shaman.ceph.com/api/search/?status=ready&project=kernel&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=nightly_pre-single-major 1 jobs: ['6486201'] suites: ['clusters/separate-client', 'conf', 'filestore-xfs', 'kernels/pre-single-major', 'krbd/unmap/{ceph/ceph', 'tasks/unmap}']