Found 184 jobs Found 8 distinct failure reasons Failure: 19 jobs: ['6682011', '6682066', '6682055', '6681915', '6681928', '6682024', '6681892', '6682031', '6681985', '6681938', '6682044', '6681889', '6681976', '6682048', '6681969', '6681931', '6681964', '6682077', '6681900'] suites intersection: ['cluster/{2-node', 'msgr-failures/few', 'openstack}'] suites union: ['clients/mirror', 'clients/{mirror-extra', 'cluster/{2-node', 'mirror}', 'msgr-failures/few', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp-lz4', 'objectstore/bluestore-comp-snappy', 'objectstore/bluestore-comp-zlib', 'objectstore/bluestore-comp-zstd', 'objectstore/bluestore-hybrid', 'objectstore/bluestore-low-osd-mem-target', 'objectstore/bluestore-stupid', 'objectstore/filestore-xfs', 'openstack}', 'policy/none', 'policy/simple', 'rbd-mirror/four-per-cluster', 'rbd/mirror-thrash/{base/install', 'rbd/mirror/{base/install', 'supported-random-distro$/{centos_8}', 'supported-random-distro$/{ubuntu_latest}', 'workloads/rbd-mirror-journal-workunit-multipeer}', 'workloads/rbd-mirror-snapshot-workunit-fast-diff-multipeer}'] Failure: "2022-02-14T15:08:27.484371+0000 mon.a (mon.0) 208 : cluster [WRN] Health check failed: Degraded data redundancy: 4/3304 objects degraded (0.121%), 1 pg degraded (PG_DEGRADED)" in cluster log 4 jobs: ['6681919', '6681901', '6681910', '6682027'] suites intersection: ['cluster/{2-node', 'msgr-failures/few', 'openstack}'] suites union: ['clients/mirror', 'clients/{mirror-extra', 'cluster/{2-node', 'mirror}', 'msgr-failures/few', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp-lz4', 'objectstore/bluestore-comp-zlib', 'objectstore/bluestore-hybrid', 'openstack}', 'policy/none', 'policy/simple', 'rbd-mirror/four-per-cluster', 'rbd/mirror-thrash/{base/install', 'rbd/mirror/{base/install', 'supported-random-distro$/{centos_8}', 'supported-random-distro$/{ubuntu_latest}', 'workloads/rbd-mirror-fsx-workunit}', 'workloads/rbd-mirror-workunit-config-key}'] Failure: reached maximum tries (90) after waiting for 540 seconds 2 jobs: ['6682040', '6682076'] suites intersection: ['clients/mirror', 'cluster/{2-node', 'msgr-failures/few', 'openstack}', 'rbd-mirror/four-per-cluster', 'rbd/mirror-thrash/{base/install'] suites union: ['clients/mirror', 'cluster/{2-node', 'msgr-failures/few', 'objectstore/bluestore-comp-zlib', 'objectstore/bluestore-stupid', 'openstack}', 'policy/none', 'policy/simple', 'rbd-mirror/four-per-cluster', 'rbd/mirror-thrash/{base/install', 'supported-random-distro$/{centos_8}', 'supported-random-distro$/{ubuntu_latest}', 'workloads/rbd-mirror-journal-stress-workunit}', 'workloads/rbd-mirror-snapshot-stress-workunit-minimum}'] Dead: b'2022-02-14T22:35:28.375 DEBUG:teuthology.exit:Finished running handlers' ceph version 17.0.0-10678-gc427b7b3 (c427b7b3d70b0814894a0b85f43e58f8da3b5e2a) quincy (dev) 1: /lib64/libpthread.so.0(+0x12c20) [0x7f8deffbdc20] 2: gsignal() 3: abort() 4: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0x1b8) [0x5598447d060e] 5: (KernelDevice::_aio_thread()+0xf9e) [0x55984541a47e] 6: (KernelDevice::AioCompletionThread::entry()+0x11) [0x559845427cb1] 7: /lib64/libpthread.so.0(+0x817f) [0x7f8deffb317f] 8: clone() 1 jobs: ['6682035'] suites: ['clients/mirror', 'cluster/{2-node', 'msgr-failures/few', 'objectstore/bluestore-hybrid', 'openstack}', 'policy/simple', 'rbd-mirror/four-per-cluster', 'rbd/mirror-thrash/{base/install', 'supported-random-distro$/{centos_8}', 'workloads/rbd-mirror-snapshot-stress-workunit-exclusive-lock}'] Dead: b'2022-02-14T21:53:51.926 DEBUG:teuthology.exit:Finished running handlers' ceph version 17.0.0-10678-gc427b7b3 (c427b7b3d70b0814894a0b85f43e58f8da3b5e2a) quincy (dev) 1: /lib64/libpthread.so.0(+0x12c20) [0x7f473db16c20] 2: pthread_cond_wait() 3: (std::condition_variable::wait(std::unique_lock&)+0x10) [0x7f473c91f8f0] 4: (rbd::mirror::LeaderWatcher::init()+0xfb) [0x56056ada46fb] 5: (rbd::mirror::PoolReplayer::init(std::__cxx11::basic_string, std::allocator > const&)+0x5a0) [0x56056ad60520] 6: (rbd::mirror::Mirror::update_pool_replayers(std::map >, std::less, std::allocator > > > > const&, std::__cxx11::basic_string, std::allocator > const&)+0x48e) [0x56056ad4bd2e] 7: (rbd::mirror::Mirror::run()+0x348) [0x56056ad4c828] 8: main() 9: __libc_start_main() 10: _start() 1 jobs: ['6681973'] suites: ['clients/mirror', 'cluster/{2-node', 'msgr-failures/few', 'objectstore/bluestore-comp-zstd', 'openstack}', 'policy/simple', 'rbd-mirror/four-per-cluster', 'rbd/mirror-thrash/{base/install', 'supported-random-distro$/{centos_8}', 'workloads/rbd-mirror-fsx-workunit}'] Failure: Command crashed: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_librbd_fsx --cluster cluster1 --id mirror.2 -d -W -R -p 100 -P /home/ubuntu/cephtest/archive -r 1 -w 1 -t 1 -h 1 -l 250000000 -S 0 -N 6000 -k mirror image_cluster1.client.mirror.2' 1 jobs: ['6682074'] suites: ['clients/mirror', 'cluster/{2-node', 'msgr-failures/few', 'objectstore/bluestore-comp-snappy', 'openstack}', 'policy/simple', 'rbd-mirror/four-per-cluster', 'rbd/mirror-thrash/{base/install', 'supported-random-distro$/{ubuntu_latest}', 'workloads/rbd-mirror-fsx-workunit}'] Failure: Command failed (workunit test rbd/rbd_mirror_stress.sh) on smithi201 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=c427b7b3d70b0814894a0b85f43e58f8da3b5e2a TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster cluster1" CEPH_ID="mirror" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_MNT=/home/ubuntu/cephtest/mnt.cluster1.mirror CEPH_ARGS=\'\' MIRROR_IMAGE_MODE=snapshot MIRROR_POOL_MODE=image RBD_IMAGE_FEATURES=layering,exclusive-lock,object-map,fast-diff RBD_MIRROR_INSTANCES=4 RBD_MIRROR_USE_EXISTING_CLUSTER=1 RBD_MIRROR_USE_RBD_MIRROR=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_stress.sh' 1 jobs: ['6681905'] suites: ['clients/mirror', 'cluster/{2-node', 'msgr-failures/few', 'objectstore/bluestore-comp-zlib', 'openstack}', 'policy/none', 'rbd-mirror/four-per-cluster', 'rbd/mirror-thrash/{base/install', 'supported-random-distro$/{centos_8}', 'workloads/rbd-mirror-snapshot-stress-workunit-fast-diff}'] Failure: Command failed (workunit test rbd/test_rbd_mirror.sh) on smithi141 with status 139: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=c427b7b3d70b0814894a0b85f43e58f8da3b5e2a TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/test_rbd_mirror.sh' 1 jobs: ['6682029'] suites: ['objectstore/filestore-xfs', 'openstack', 'rbd/singleton/{all/rbd_mirror', 'supported-random-distro$/{ubuntu_latest}}']