Found 32 jobs Found 4 distinct failure reasons Failure: Command failed (workunit test rbd/rbd_mirror_snapshot.sh) on smithi181 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=bf9111326e473b2d564ee1e0cc947d0a6f09424c TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster cluster1" CEPH_ID="mirror" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_MNT=/home/ubuntu/cephtest/mnt.cluster1.mirror CEPH_ARGS=\'\' RBD_IMAGE_FEATURES=layering,exclusive-lock,object-map,fast-diff RBD_MIRROR_CONFIG_KEY=1 RBD_MIRROR_INSTANCES=4 RBD_MIRROR_USE_EXISTING_CLUSTER=1 RBD_MIRROR_USE_RBD_MIRROR=1 USE_CLUSTER3=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_snapshot.sh' 16 jobs: ['6884325', '6884339', '6884317', '6884346', '6884318', '6884329', '6884330', '6884320', '6884327', '6884343', '6884334', '6884337', '6884321', '6884342', '6884323', '6884332'] suites intersection: ['cluster/3-nodes/{base/{install', 'mirror', 'msgr-failures/few', 'roles}', 'workloads/rbd-mirror-snapshot-workunit-fast-diff-multipeer}'] suites union: ['cluster/3-nodes/{base/{install', 'extra/{mirror-extra}', 'extra/{rbd-mirror}', 'mirror', 'msgr-failures/few', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp-lz4', 'objectstore/bluestore-comp-snappy', 'objectstore/bluestore-comp-zlib', 'objectstore/bluestore-comp-zstd', 'objectstore/bluestore-low-osd-mem-target', 'objectstore/bluestore-stupid', 'objectstore/filestore-xfs', 'policy/none', 'policy/simple', 'rbd/mirror-thrash/{base/{openstack}', 'rbd/mirror/{base/{openstack}', 'roles}', 'supported-random-distro$/{centos_8}}', 'supported-random-distro$/{ubuntu_latest}}', 'workloads/rbd-mirror-snapshot-workunit-fast-diff-multipeer}'] Failure: Command failed (workunit test rbd/rbd_mirror_journal.sh) on smithi096 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=bf9111326e473b2d564ee1e0cc947d0a6f09424c TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster cluster1" CEPH_ID="mirror" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_MNT=/home/ubuntu/cephtest/mnt.cluster1.mirror CEPH_ARGS=\'\' RBD_MIRROR_INSTANCES=4 RBD_MIRROR_USE_EXISTING_CLUSTER=1 RBD_MIRROR_USE_RBD_MIRROR=1 USE_CLUSTER3=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_journal.sh' 5 jobs: ['6884335', '6884322', '6884316', '6884315', '6884328'] suites intersection: ['cluster/3-nodes/{base/{install', 'extra/{rbd-mirror}', 'mirror', 'msgr-failures/few', 'rbd/mirror-thrash/{base/{openstack}', 'roles}', 'workloads/rbd-mirror-journal-workunit-multipeer}'] suites union: ['cluster/3-nodes/{base/{install', 'extra/{rbd-mirror}', 'mirror', 'msgr-failures/few', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp-lz4', 'objectstore/bluestore-comp-zlib', 'objectstore/bluestore-low-osd-mem-target', 'objectstore/bluestore-stupid', 'policy/none', 'policy/simple', 'rbd/mirror-thrash/{base/{openstack}', 'roles}', 'supported-random-distro$/{centos_8}}', 'supported-random-distro$/{ubuntu_latest}}', 'workloads/rbd-mirror-journal-workunit-multipeer}'] Failure: reached maximum tries (50) after waiting for 300 seconds 1 jobs: ['6884333'] suites: ['cluster/3-nodes/{base/{install', 'extra/{rbd-mirror}', 'mirror', 'msgr-failures/few', 'objectstore/filestore-xfs', 'policy/none', 'rbd/mirror-thrash/{base/{openstack}', 'roles}', 'supported-random-distro$/{centos_8}}', 'workloads/rbd-mirror-journal-workunit-multipeer}'] Failure: "2022-06-17T13:38:26.032989+0000 mon.a (mon.0) 99 : cluster [WRN] Health check failed: Degraded data redundancy: 2/4 objects degraded (50.000%), 1 pg degraded (PG_DEGRADED)" in cluster log 1 jobs: ['6884331'] suites: ['cluster/3-nodes/{base/{install', 'extra/{rbd-mirror}', 'mirror', 'msgr-failures/few', 'objectstore/bluestore-hybrid', 'policy/none', 'rbd/mirror-thrash/{base/{openstack}', 'roles}', 'supported-random-distro$/{ubuntu_latest}}', 'workloads/rbd-mirror-snapshot-workunit-fast-diff-multipeer}']