Found 178 jobs Missing teuthology log /home/teuthworker/archive/teuthology-2022-06-13_06:07:05-rbd-pacific-distro-default-smithi/6876225/teuthology.log Found 6 distinct failure reasons Timeout 3h running clone.client.0/qa/workunits/rbd/cli_generic.sh 6 jobs: ['6876158', '6876189', '6876278', '6876310', '6876218', '6876247'] suites intersection: ['clusters/{fixed-1', 'features/format-1', 'msgr-failures/few', 'openstack}', 'pool/small-cache-pool', 'rbd/cli_v1/{base/install', 'workloads/rbd_cli_generic}'] suites union: ['clusters/{fixed-1', 'features/format-1', 'msgr-failures/few', 'objectstore/bluestore-comp-lz4', 'objectstore/bluestore-comp-zlib', 'objectstore/bluestore-comp-zstd', 'objectstore/bluestore-hybrid', 'objectstore/bluestore-low-osd-mem-target', 'objectstore/filestore-xfs', 'openstack}', 'pool/small-cache-pool', 'rbd/cli_v1/{base/install', 'supported-random-distro$/{centos_8}', 'supported-random-distro$/{rhel_8}', 'supported-random-distro$/{ubuntu_latest}', 'workloads/rbd_cli_generic}'] Failure: Command failed on smithi061 with status 151: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_librbd_fsx --cluster ceph --id 0 -d -W -R -p 100 -P /home/ubuntu/cephtest/archive -r 512 -w 512 -t 512 -h 512 -l 250000000 -S 0 -N 6000 -M pool_client.0 image_client.0' 1 jobs: ['6876159'] suites: ['cluster/{fixed-3', 'msgr-failures/few', 'objectstore/bluestore-comp-zstd', 'openstack}', 'rbd/nbd/{base/install', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/default', 'thrashosds-health', 'workloads/rbd_fsx_nbd}'] Dead: None 1 jobs: ['6876225'] Failure: "2022-06-13T07:06:18.600369+0000 mon.a (mon.0) 197 : cluster [WRN] Health check failed: Degraded data redundancy: 3/330 objects degraded (0.909%), 1 pg degraded (PG_DEGRADED)" in cluster log 1 jobs: ['6876226'] suites: ['clients/mirror', 'cluster/{2-node', 'msgr-failures/few', 'objectstore/bluestore-comp-zlib', 'openstack}', 'policy/simple', 'rbd-mirror/four-per-cluster', 'rbd/mirror-thrash/{base/install', 'supported-random-distro$/{ubuntu_latest}', 'workloads/rbd-mirror-snapshot-stress-workunit-exclusive-lock}'] Timeout 3h running clone.client.0/qa/workunits/rbd/qemu_dynamic_features.sh 2 jobs: ['6876265', '6876227'] suites intersection: ['clusters/{fixed-3', 'openstack}', 'qemu/xfstests', 'rbd/maintenance/{base/install', 'supported-random-distro$/{ubuntu_latest}', 'workloads/dynamic_features_no_cache}'] suites union: ['clusters/{fixed-3', 'objectstore/bluestore-comp-zstd', 'objectstore/bluestore-stupid', 'openstack}', 'qemu/xfstests', 'rbd/maintenance/{base/install', 'supported-random-distro$/{ubuntu_latest}', 'workloads/dynamic_features_no_cache}'] Failure: Command failed (workunit test rbd/rbd_mirror_snapshot.sh) on smithi103 with status 16: 'mkdir -p -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=c0a5b05641f696a6edd94f72904e922da2523ba6 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster cluster1" CEPH_ID="mirror" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_MNT=/home/ubuntu/cephtest/mnt.cluster1.mirror CEPH_ARGS=\'\' RBD_IMAGE_FEATURES=layering RBD_MIRROR_CONFIG_KEY=1 RBD_MIRROR_INSTANCES=4 RBD_MIRROR_USE_EXISTING_CLUSTER=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_snapshot.sh' 1 jobs: ['6876214'] suites: ['clients/{mirror-extra', 'cluster/{2-node', 'mirror}', 'msgr-failures/few', 'objectstore/bluestore-comp-zlib', 'openstack}', 'rbd/mirror/{base/install', 'supported-random-distro$/{rhel_8}', 'workloads/rbd-mirror-snapshot-workunit-minimum}']