Found 18 jobs
Found 6 distinct failure reasons
Failure: reached maximum tries (90) after waiting for 540 seconds
2 jobs: ['6779090', '6779095']
suites intersection: ['cluster/3-nodes/{base/{install', 'extra/{rbd-mirror}', 'mirror', 'msgr-failures/few', 'policy/none', 'rbd/mirror-thrash/{base/{openstack}', 'roles}', 'workloads/rbd-mirror-journal-workunit-multipeer}']
suites union: ['cluster/3-nodes/{base/{install', 'extra/{rbd-mirror}', 'mirror', 'msgr-failures/few', 'objectstore/bluestore-comp-lz4', 'objectstore/bluestore-comp-snappy', 'policy/none', 'rbd/mirror-thrash/{base/{openstack}', 'roles}', 'supported-random-distro$/{centos_8}}', 'supported-random-distro$/{ubuntu_latest}}', 'workloads/rbd-mirror-journal-workunit-multipeer}']

Failure: Command failed (workunit test rbd/rbd_mirror_journal.sh) on smithi156 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=46d4842c29bb1566ea5a5be2b85263044364e053 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster cluster1" CEPH_ID="mirror" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_MNT=/home/ubuntu/cephtest/mnt.cluster1.mirror CEPH_ARGS=\'\' RBD_MIRROR_INSTANCES=4 RBD_MIRROR_USE_EXISTING_CLUSTER=1 RBD_MIRROR_USE_RBD_MIRROR=1 USE_CLUSTER3=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_journal.sh'
5 jobs: ['6779088', '6779091', '6779081', '6779084', '6779085']
suites intersection: ['cluster/3-nodes/{base/{install', 'extra/{rbd-mirror}', 'mirror', 'msgr-failures/few', 'rbd/mirror-thrash/{base/{openstack}', 'roles}', 'workloads/rbd-mirror-journal-workunit-multipeer}']
suites union: ['cluster/3-nodes/{base/{install', 'extra/{rbd-mirror}', 'mirror', 'msgr-failures/few', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp-zlib', 'objectstore/bluestore-comp-zstd', 'objectstore/filestore-xfs', 'policy/none', 'policy/simple', 'rbd/mirror-thrash/{base/{openstack}', 'roles}', 'supported-random-distro$/{centos_8}}', 'supported-random-distro$/{ubuntu_latest}}', 'workloads/rbd-mirror-journal-workunit-multipeer}']

Crash: reached maximum tries (50) after waiting for 300 seconds
ceph version 17.0.0-11492-g46d4842c (46d4842c29bb1566ea5a5be2b85263044364e053) quincy (dev)
 1: /lib64/libpthread.so.0(+0x12ce0) [0x7f3e9e70cce0]
 2: /lib64/libpthread.so.0(+0x1181d) [0x7f3e9e70b81d]
 3: __pthread_mutex_lock()
 4: (rbd::mirror::NamespaceReplayer<librbd::ImageCtx>::add_peer(rbd::mirror::Peer<librbd::ImageCtx> const&, Context*)+0x18b) [0x55cbd04f27cb]
 5: (rbd::mirror::PoolReplayer<librbd::ImageCtx>::add_peer(rbd::mirror::PeerSpec const&, C_GatherBase<Context, Context>*)+0x9fb) [0x55cbd048c45b]
 6: (rbd::mirror::PoolReplayer<librbd::ImageCtx>::update_peers(std::set<rbd::mirror::PeerSpec, rbd::mirror::ClusterWatcher::PeerSpecCompare, std::allocator<rbd::mirror::PeerSpec> > const&)+0x5df) [0x55cbd048fddf]
 7: (rbd::mirror::Mirror::update_pool_replayers(std::map<long, std::set<rbd::mirror::PeerSpec, rbd::mirror::ClusterWatcher::PeerSpecCompare, std::allocator<rbd::mirror::PeerSpec> >, std::less<long>, std::allocator<std::pair<long const, std::set<rbd::mirror::PeerSpec, rbd::mirror::ClusterWatcher::PeerSpecCompare, std::allocator<rbd::mirror::PeerSpec> > > > > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x72b) [0x55cbd047c25b]
 8: (rbd::mirror::Mirror::run()+0x348) [0x55cbd047cab8]
 9: main()
 10: __libc_start_main()
 11: _start()
1 jobs: ['6779082']
suites: ['cluster/3-nodes/{base/{install', 'extra/{rbd-mirror}', 'mirror', 'msgr-failures/few', 'objectstore/bluestore-low-osd-mem-target', 'policy/simple', 'rbd/mirror-thrash/{base/{openstack}', 'roles}', 'supported-random-distro$/{centos_8}}', 'workloads/rbd-mirror-journal-workunit-multipeer}']

Failure: "2022-04-06T13:05:57.310287+0000 mon.a (mon.0) 96 : cluster [WRN] Health check failed: Degraded data redundancy: 2/4 objects degraded (50.000%), 1 pg degraded (PG_DEGRADED)" in cluster log
1 jobs: ['6779086']
suites: ['cluster/3-nodes/{base/{install', 'extra/{rbd-mirror}', 'mirror', 'msgr-failures/few', 'objectstore/bluestore-stupid', 'policy/simple', 'rbd/mirror-thrash/{base/{openstack}', 'roles}', 'supported-random-distro$/{ubuntu_latest}}', 'workloads/rbd-mirror-journal-workunit-multipeer}']

Dead: b'2022-04-07T10:47:25.894 DEBUG:teuthology.exit:Finished running handlers'
ceph version 17.0.0-11492-g46d4842c (46d4842c29bb1566ea5a5be2b85263044364e053) quincy (dev)
 1: /lib64/libpthread.so.0(+0x12ce0) [0x7fe041618ce0]
 2: pthread_cond_wait()
 3: (std::condition_variable::wait(std::unique_lock<std::mutex>&)+0x10) [0x7fe0404218f0]
 4: (rbd::mirror::LeaderWatcher<librbd::ImageCtx>::shut_down()+0xfb) [0x56068043d5bb]
 5: (rbd::mirror::PoolReplayer<librbd::ImageCtx>::shut_down()+0x87) [0x5606803f7307]
 6: (rbd::mirror::PoolReplayer<librbd::ImageCtx>::~PoolReplayer()+0x10) [0x5606803f7630]
 7: (std::_Rb_tree<long, std::pair<long const, std::unique_ptr<rbd::mirror::PoolReplayer<librbd::ImageCtx>, std::default_delete<rbd::mirror::PoolReplayer<librbd::ImageCtx> > > >, std::_Select1st<std::pair<long const, std::unique_ptr<rbd::mirror::PoolReplayer<librbd::ImageCtx>, std::default_delete<rbd::mirror::PoolReplayer<librbd::ImageCtx> > > > >, std::less<long>, std::allocator<std::pair<long const, std::unique_ptr<rbd::mirror::PoolReplayer<librbd::ImageCtx>, std::default_delete<rbd::mirror::PoolReplayer<librbd::ImageCtx> > > > > >::_M_erase(std::_Rb_tree_node<std::pair<long const, std::unique_ptr<rbd::mirror::PoolReplayer<librbd::ImageCtx>, std::default_delete<rbd::mirror::PoolReplayer<librbd::ImageCtx> > > > >*)+0x3a) [0x5606803e5eca]
 8: (std::_Rb_tree<long, std::pair<long const, std::unique_ptr<rbd::mirror::PoolReplayer<librbd::ImageCtx>, std::default_delete<rbd::mirror::PoolReplayer<librbd::ImageCtx> > > >, std::_Select1st<std::pair<long const, std::unique_ptr<rbd::mirror::PoolReplayer<librbd::ImageCtx>, std::default_delete<rbd::mirror::PoolReplayer<librbd::ImageCtx> > > > >, std::less<long>, std::allocator<std::pair<long const, std::unique_ptr<rbd::mirror::PoolReplayer<librbd::ImageCtx>, std::default_delete<rbd::mirror::PoolReplayer<librbd::ImageCtx> > > > > >::_M_erase(std::_Rb_tree_node<std::pair<long const, std::unique_ptr<rbd::mirror::PoolReplayer<librbd::ImageCtx>, std::default_delete<rbd::mirror::PoolReplayer<librbd::ImageCtx> > > > >*)+0x25) [0x5606803e5eb5]
 9: (rbd::mirror::Mirror::~Mirror()+0xc5) [0x5606803e0af5]
 10: main()
 11: __libc_start_main()
 12: _start()
2 jobs: ['6779098', '6779093']
suites intersection: ['cluster/3-nodes/{base/{install', 'extra/{rbd-mirror}', 'mirror', 'msgr-failures/few', 'rbd/mirror-thrash/{base/{openstack}', 'roles}', 'supported-random-distro$/{centos_8}}', 'workloads/rbd-mirror-journal-workunit-multipeer}']
suites union: ['cluster/3-nodes/{base/{install', 'extra/{rbd-mirror}', 'mirror', 'msgr-failures/few', 'objectstore/bluestore-comp-zlib', 'objectstore/bluestore-comp-zstd', 'policy/none', 'policy/simple', 'rbd/mirror-thrash/{base/{openstack}', 'roles}', 'supported-random-distro$/{centos_8}}', 'workloads/rbd-mirror-journal-workunit-multipeer}']

Dead: b'2022-04-06T19:29:41.153 DEBUG:teuthology.exit:Finished running handlers'
ceph version 17.0.0-11492-g46d4842c (46d4842c29bb1566ea5a5be2b85263044364e053) quincy (dev)
 1: /lib64/libpthread.so.0(+0x12ce0) [0x7f177351dce0]
 2: gsignal()
 3: abort()
 4: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1b0) [0x7f17740eb192]
 5: /usr/lib64/ceph/libceph-common.so.2(+0x284355) [0x7f17740eb355]
 6: (rbd::mirror::NamespaceReplayer<librbd::ImageCtx>::add_peer(rbd::mirror::Peer<librbd::ImageCtx> const&, Context*)+0x3f2) [0x55ac51bd4a32]
 7: (rbd::mirror::PoolReplayer<librbd::ImageCtx>::update_namespace_replayers()+0x451) [0x55ac51b73f61]
 8: (rbd::mirror::PoolReplayer<librbd::ImageCtx>::run()+0x9b3) [0x55ac51b75213]
 9: (rbd::mirror::PoolReplayer<librbd::ImageCtx>::PoolReplayerThread::entry()+0x11) [0x55ac51b75e01]
 10: /lib64/libpthread.so.0(+0x81cf) [0x7f17735131cf]
 11: clone()
1 jobs: ['6779087']
suites: ['cluster/3-nodes/{base/{install', 'extra/{rbd-mirror}', 'mirror', 'msgr-failures/few', 'objectstore/bluestore-hybrid', 'policy/none', 'rbd/mirror-thrash/{base/{openstack}', 'roles}', 'supported-random-distro$/{centos_8}}', 'workloads/rbd-mirror-journal-workunit-multipeer}']