Found 222 jobs Found 7 distinct failure reasons Failure: Failed to fetch package version from https://shaman.ceph.com/api/search/?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=aa5f213d09bc881fe4dbd70c4e27946872a54f53 97 jobs suites intersection: ['conf'] suites union: ['ceph/ceph', 'clusters/3-node', 'clusters/fixed-1', 'clusters/fixed-3', 'clusters/separate-client', 'conf', 'features/no-object-map', 'features/object-map', 'filestore-xfs', 'install/ceph', 'kernels/pre-single-major', 'krbd/basic/{bluestore-bitmap', 'krbd/fsx/{ceph/ceph', 'krbd/ms_modeless/{bluestore-bitmap', 'krbd/rbd-nomount/{bluestore-bitmap', 'krbd/rbd/{bluestore-bitmap', 'krbd/singleton/{bluestore-bitmap', 'krbd/thrash/{bluestore-bitmap', 'krbd/unmap/{ceph/ceph', 'krbd/wac/wac/{bluestore-bitmap', 'ms_mode$/{crc-rxbounce}', 'ms_mode$/{crc}', 'ms_mode$/{legacy-rxbounce}', 'ms_mode$/{legacy}', 'ms_mode$/{secure}', 'ms_mode/crc$/{crc-rxbounce}', 'ms_mode/crc$/{crc}', 'ms_mode/legacy$/{legacy-rxbounce}', 'ms_mode/legacy$/{legacy}', 'ms_mode/secure', 'msgr-failures/few', 'msgr-failures/many', 'objectstore/bluestore-bitmap', 'objectstore/filestore-xfs', 'randomized-striping-off}', 'randomized-striping-on}', 'striping/default/{msgr-failures/few', 'striping/default/{msgr-failures/many', 'striping/fancy/{msgr-failures/few', 'tasks/fsx-1-client}', 'tasks/fsx-3-client}', 'tasks/krbd_data_pool}', 'tasks/krbd_discard}', 'tasks/krbd_exclusive_option}', 'tasks/krbd_huge_image}', 'tasks/krbd_huge_osdmap}', 'tasks/krbd_latest_osdmap_on_map}', 'tasks/krbd_namespaces}', 'tasks/krbd_parent_overlap}', 'tasks/krbd_read_only}', 'tasks/krbd_rxbounce}', 'tasks/krbd_udev_enumerate}', 'tasks/krbd_udev_netlink_enobufs}', 'tasks/krbd_udev_netns}', 'tasks/krbd_udev_symlinks}', 'tasks/rbd_concurrent}', 'tasks/rbd_fio}', 'tasks/rbd_huge_tickets}', 'tasks/rbd_image_read}', 'tasks/rbd_kernel}', 'tasks/rbd_map_snapshot_io}', 'tasks/rbd_map_unmap}', 'tasks/rbd_simple_big}', 'tasks/rbd_workunit_kernel_untar_build}', 'tasks/rbd_workunit_suites_dbench}', 'tasks/rbd_workunit_suites_ffsb}', 'tasks/rbd_workunit_suites_fsstress_ext4}', 'tasks/rbd_workunit_suites_fsstress}', 'tasks/rbd_workunit_suites_fsx}', 'tasks/rbd_workunit_suites_iozone}', 'tasks/rbd_workunit_trivial_sync}', 'tasks/rbd_xfstests}', 'tasks/unmap}', 'tasks/wac', 'thrashers/backoff', 'thrashers/upmap', 'thrashosds-health', 'verify/no-resets}', 'workloads/rbd_workunit_suites_ffsb}'] Failure: Command failed (workunit test kernel_untar_build.sh) on smithi203 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=9613de7fd7989a333961e90d9306d8abf1d24564 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/kernel_untar_build.sh' 1 jobs: ['7054178'] suites: ['clusters/fixed-3', 'conf', 'krbd/rbd/{bluestore-bitmap', 'ms_mode/secure', 'msgr-failures/many', 'tasks/rbd_workunit_kernel_untar_build}'] Failure: Command failed on smithi115 with status 2: 'cd /home/ubuntu/cephtest/rbd_fio_test ; wget https://github.com/axboe/fio/archive/fio-3.16.tar.gz ; tar -xvf fio*tar.gz ; cd fio-fio* ; ./configure ; make' 8 jobs: ['7054285', '7054102', '7054099', '7054295', '7054239', '7054156', '7054138', '7054173'] suites intersection: ['clusters/fixed-3', 'conf'] suites union: ['ceph/ceph', 'clusters/fixed-3', 'conf', 'krbd/rbd/{bluestore-bitmap', 'krbd/thrash/{bluestore-bitmap', 'ms_mode$/{crc-rxbounce}', 'ms_mode$/{crc}', 'ms_mode$/{legacy-rxbounce}', 'ms_mode/crc$/{crc}', 'ms_mode/legacy$/{legacy-rxbounce}', 'ms_mode/legacy$/{legacy}', 'ms_mode/secure', 'msgr-failures/few', 'msgr-failures/many', 'tasks/rbd_fio}', 'thrashers/backoff', 'thrashers/mon-thrasher', 'thrashers/pggrow', 'thrashers/upmap', 'thrashosds-health', 'workloads/rbd_fio}'] Failure: Command failed on smithi172 with status 166: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_librbd_fsx --cluster ceph --id 0 -d -W -R -p 100 -P /home/ubuntu/cephtest/archive -r 512 -w 512 -t 512 -h 512 -l 250000000 -S 0 -N 10000 -K -U pool_client.0 image_client.0' 2 jobs: ['7054147', '7054303'] suites intersection: ['clusters/3-node', 'conf', 'features/object-map', 'krbd/fsx/{ceph/ceph', 'objectstore/filestore-xfs'] suites union: ['clusters/3-node', 'conf', 'features/object-map', 'krbd/fsx/{ceph/ceph', 'ms_mode$/{crc}', 'ms_mode$/{legacy-rxbounce}', 'objectstore/filestore-xfs', 'randomized-striping-off}', 'randomized-striping-on}', 'striping/default/{msgr-failures/many', 'striping/fancy/{msgr-failures/few', 'tasks/fsx-1-client}', 'tasks/fsx-3-client}'] Failure: "1664928032.7409065 mon.a (mon.0) 276 : cluster [WRN] Health check failed: 0 slow ops, oldest one blocked for 33 sec, osd.5 has slow ops (SLOW_OPS)" in cluster log 1 jobs: ['7054219'] suites: ['clusters/fixed-3', 'conf', 'krbd/rbd/{bluestore-bitmap', 'ms_mode/secure', 'msgr-failures/many', 'tasks/rbd_workunit_suites_dbench}'] Failure: Cannot connect to remote host smithi103 1 jobs: ['7054233'] suites: ['clusters/fixed-3', 'conf', 'install/ceph', 'krbd/rbd-nomount/{bluestore-bitmap', 'ms_mode/legacy$/{legacy}', 'msgr-failures/many', 'tasks/rbd_concurrent}'] Failure: Command failed on smithi150 with status 2: "/usr/bin/sudo TESTDIR=/home/ubuntu/cephtest adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage /bin/bash /home/ubuntu/cephtest/run_xfstests.sh -c 1 -f ext4 -t /dev/rbd0 -s /dev/rbd1 -x /tmp/excludeaw4_yqea -r -- '-g auto -g blockdev -x clone'" 1 jobs: ['7054213'] suites: ['conf', 'krbd/singleton/{bluestore-bitmap', 'ms_mode$/{legacy}', 'msgr-failures/many', 'tasks/rbd_xfstests}']