Found 100 jobs Found 5 distinct failure reasons Crash: error during thrashing ceph version 16.0.0-6204-g37cc998b (37cc998bad8b922eb928bdbbd0ac8369ef18f5f5) pacific (dev) 1: /lib64/libpthread.so.0(+0x12dc0) [0x7f7c0c91ddc0] 2: pthread_cond_wait() 3: (std::condition_variable::wait(std::unique_lock&)+0x10) [0x7f7c0b79a870] 4: (rbd::mirror::LeaderWatcher::shut_down()+0xfb) [0x560ed37ca62b] 5: (rbd::mirror::PoolReplayer::shut_down()+0x8a) [0x560ed3785aea] 6: (rbd::mirror::PoolReplayer::~PoolReplayer()+0x12) [0x560ed3785d42] 7: (std::_Rb_tree, std::pair const, std::unique_ptr, std::default_delete > > >, std::_Select1st const, std::unique_ptr, std::default_delete > > > >, std::less >, std::allocator const, std::unique_ptr, std::default_delete > > > > >::_M_erase(std::_Rb_tree_node const, std::unique_ptr, std::default_delete > > > >*)+0x41) [0x560ed377de81] 8: (std::_Rb_tree, std::pair const, std::unique_ptr, std::default_delete > > >, std::_Select1st const, std::unique_ptr, std::default_delete > > > >, std::less >, std::allocator const, std::unique_ptr, std::default_delete > > > > >::_M_erase(std::_Rb_tree_node const, std::unique_ptr, std::default_delete > > > >*)+0x29) [0x560ed377de69] 9: (rbd::mirror::Mirror::~Mirror()+0xc5) [0x560ed3777ea5] 10: main() 11: __libc_start_main() 12: _start() 1 jobs: ['5509455'] suites: ['clients/mirror', 'cluster/{2-node', 'msgr-failures/few', 'objectstore/bluestore-comp-zlib', 'openstack}', 'policy/none', 'rbd-mirror/four-per-cluster', 'rbd/mirror-thrash/{base/install', 'supported-random-distro$/{rhel_8}', 'workloads/rbd-mirror-fsx-workunit}'] Failure: Command failed (workunit test rbd/rbd_mirror_fsx_compare.sh) on smithi124 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=37cc998bad8b922eb928bdbbd0ac8369ef18f5f5 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster cluster1" CEPH_ID="mirror" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_ARGS=\'\' RBD_MIRROR_USE_EXISTING_CLUSTER=1 RBD_MIRROR_USE_RBD_MIRROR=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_fsx_compare.sh' 1 jobs: ['5509368'] suites: ['clients/mirror', 'cluster/{2-node', 'msgr-failures/few', 'objectstore/bluestore-comp-zlib', 'openstack}', 'policy/simple', 'rbd-mirror/four-per-cluster', 'rbd/mirror-thrash/{base/install', 'supported-random-distro$/{rhel_8}', 'workloads/rbd-mirror-fsx-workunit}'] Failure: Command crashed: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_librbd_fsx --cluster cluster1 --id mirror.1 -d -W -R -p 100 -P /home/ubuntu/cephtest/archive -r 1 -w 1 -t 1 -h 1 -l 250000000 -S 0 -N 6000 -k mirror image_cluster1.client.mirror.1' 1 jobs: ['5509391'] suites: ['clients/mirror', 'cluster/{2-node', 'msgr-failures/few', 'objectstore/bluestore-comp-lz4', 'openstack}', 'policy/none', 'rbd-mirror/four-per-cluster', 'rbd/mirror-thrash/{base/install', 'supported-random-distro$/{rhel_8}', 'workloads/rbd-mirror-fsx-workunit}'] Failure: "2020-10-11T15:17:42.223028+0000 mon.a (mon.0) 194 : cluster [WRN] Health check failed: Degraded data redundancy: 1/2942 objects degraded (0.034%), 1 pg degraded (PG_DEGRADED)" in cluster log 1 jobs: ['5509422'] suites: ['clients/mirror', 'cluster/{2-node', 'msgr-failures/few', 'objectstore/bluestore-comp-snappy', 'openstack}', 'policy/none', 'rbd-mirror/four-per-cluster', 'rbd/mirror-thrash/{base/install', 'supported-random-distro$/{ubuntu_latest}', 'workloads/rbd-mirror-fsx-workunit}'] Failure: Command failed (workunit test rbd/rbd_mirror_bootstrap.sh) on smithi143 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=37cc998bad8b922eb928bdbbd0ac8369ef18f5f5 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster cluster1" CEPH_ID="mirror" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_ARGS=\'\' RBD_MIRROR_INSTANCES=1 RBD_MIRROR_USE_EXISTING_CLUSTER=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_bootstrap.sh' 1 jobs: ['5509416'] suites: ['clients/{mirror-extra', 'cluster/{2-node', 'mirror}', 'msgr-failures/few', 'objectstore/bluestore-low-osd-mem-target', 'openstack}', 'rbd/mirror/{base/install', 'supported-random-distro$/{rhel_8}', 'workloads/rbd-mirror-bootstrap-workunit}']