Found 140 jobs Missing teuthology log /home/teuthworker/archive/teuthology-2021-06-28_06:07:01-rbd-pacific-distro-basic-smithi/6239313/teuthology.log Found 5 distinct failure reasons Failure: Command failed (workunit test rbd/rbd-nbd.sh) on smithi159 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=ad023bdd10d569be962277ef23fa5035f9219de0 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/rbd-nbd.sh' 1 jobs: ['6239209'] suites: ['cluster/{fixed-3', 'msgr-failures/few', 'objectstore/bluestore-comp-zstd', 'openstack}', 'rbd/nbd/{base/install', 'supported-random-distro$/{rhel_8}', 'thrashers/default', 'thrashosds-health', 'workloads/rbd_nbd}'] Dead: None 5 jobs: ['6239313', '6239360', '6239226', '6239276', '6239339'] Failure: Command failed on smithi159 with status 120: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_librbd_fsx --cluster ceph --id 0 -d -W -R -p 100 -P /home/ubuntu/cephtest/archive -r 512 -w 512 -t 512 -h 512 -l 250000000 -S 0 -N 6000 -M pool_client.0 image_client.0' 1 jobs: ['6239343'] suites: ['cluster/{fixed-3', 'msgr-failures/few', 'objectstore/bluestore-comp-zstd', 'openstack}', 'rbd/nbd/{base/install', 'supported-random-distro$/{centos_8}', 'thrashers/default', 'thrashosds-health', 'workloads/rbd_fsx_nbd}'] Failure: "2021-07-01T19:10:13.790979+0000 osd.1 (osd.1) 3 : cluster [WRN] slow request osd_op(client.4507.0:52632 2.6 2:71082d81:::rbd_data.117a2c18a5cc.0000000000000339:head [stat,set-alloc-hint object_size 4194304 write_size 4194304,write 135168~4059136 in=4059136b] snapc 0=[] ondisk+write+known_if_redirected e24) initiated 2021-07-01T19:09:43.354546+0000 currently delayed" in cluster log 1 jobs: ['6239292'] suites: ['clusters/{fixed-3', 'objectstore/bluestore-low-osd-mem-target', 'openstack}', 'qemu/xfstests', 'rbd/maintenance/{base/install', 'supported-random-distro$/{centos_8}', 'workloads/dynamic_features_no_cache}'] Failure: "2021-07-01T19:38:23.748590+0000 mon.a (mon.0) 233 : cluster [WRN] Health check failed: 0 slow ops, oldest one blocked for 30 sec, osd.1 has slow ops (SLOW_OPS)" in cluster log 1 jobs: ['6239335'] suites: ['clusters/{fixed-1', 'features/layering', 'msgr-failures/few', 'objectstore/bluestore-comp-zlib', 'openstack}', 'pool/small-cache-pool', 'rbd/cli/{base/install', 'supported-random-distro$/{centos_8}', 'workloads/rbd_cli_migration}']