Found 250 jobs Missing teuthology log /home/teuthworker/archive/dis-2023-12-10_16:10:42-rbd-wip-dis-testing-1-distro-default-smithi/7486216/teuthology.log Found 10 distinct failure reasons Dead: None 5 jobs: ['7486216', '7486332', '7486213', '7486214', '7486245'] Failure: ['MODULE FAILURE', 'See stdout/stderr for the exact error'] 1 jobs: ['7486220'] suites: ['clusters/{fixed-1', 'conf/{disable-pool-app}', 'data-pool/ec', 'features/layering', 'msgr-failures/few', 'objectstore/bluestore-low-osd-mem-target', 'openstack}', 'rbd/cli/{base/install', 'supported-random-distro$/{ubuntu_latest}', 'workloads/rbd_cli_groups}'] Failure: Command failed (workunit test rbd/test_librbd_python.sh) on smithi141 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=3d2db44fa991bc355074aa9b7647f63b77201fef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_FEATURES=125 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/test_librbd_python.sh' 21 jobs: ['7486428', '7486426', '7486338', '7486275', '7486420', '7486381', '7486297', '7486203', '7486384', '7486339', '7486315', '7486254', '7486346', '7486301', '7486379', '7486251', '7486396', '7486257', '7486210', '7486290', '7486292'] suites intersection: ['conf/{disable-pool-app}', 'openstack}'] suites union: ['centos_latest', 'clusters/{fixed-1', 'clusters/{fixed-3', 'conf/{disable-pool-app}', 'data-pool/ec', 'data-pool/none', 'data-pool/replicated', 'extra-conf/copy-on-read', 'extra-conf/none', 'extra-conf/permit-partial-discard', 'min-compat-client/default', 'min-compat-client/octopus', 'msgr-failures/few', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp-lz4', 'objectstore/bluestore-comp-snappy', 'objectstore/bluestore-comp-zlib', 'objectstore/bluestore-comp-zstd', 'objectstore/bluestore-hybrid', 'objectstore/bluestore-low-osd-mem-target', 'objectstore/bluestore-stupid', 'openstack}', 'rbd/basic/{base/install', 'rbd/librbd/{cache/none', 'rbd/librbd/{cache/writeback', 'rbd/librbd/{cache/writethrough', 'rbd/valgrind/{base/install', 'supported-random-distro$/{centos_latest}', 'supported-random-distro$/{ubuntu_latest}', 'tasks/rbd_python_api_tests_old_format}', 'validator/memcheck', 'workloads/python_api_tests_with_defaults}', 'workloads/python_api_tests_with_journaling}', 'workloads/python_api_tests}'] Failure: SSH connection to smithi072 was lost: 'sudo apt-get update' 2 jobs: ['7486221', '7486333'] suites intersection: ['conf/{disable-pool-app}'] suites union: ['clusters/{fixed-3', 'conf/{disable-pool-app}', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-low-osd-mem-target', 'openstack', 'openstack}', 'qemu/xfstests', 'rbd/maintenance/{base/install', 'rbd/singleton/{all/read-flags-no-cache', 'supported-random-distro$/{ubuntu_latest}', 'supported-random-distro$/{ubuntu_latest}}', 'workloads/dynamic_features_no_cache}'] Failure: Command failed on smithi032 with status 1: 'CEPH_REF=master CEPH_ID="0" PATH=$PATH:/usr/sbin adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage /home/ubuntu/cephtest/virtualenv/bin/cram -v -- /home/ubuntu/cephtest/archive/cram.client.0/*.t' 1 jobs: ['7486365'] suites: ['conf/{disable-pool-app}', 'objectstore/bluestore-comp-zlib', 'openstack', 'rbd/singleton/{all/snap-diff', 'supported-random-distro$/{ubuntu_latest}}'] Crash: Command failed (workunit test rbd/rbd_mirror_stress.sh) on smithi076 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=3d2db44fa991bc355074aa9b7647f63b77201fef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster cluster1" CEPH_ID="mirror" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_MNT=/home/ubuntu/cephtest/mnt.cluster1.mirror CEPH_ARGS=\'\' MIRROR_POOL_MODE=image RBD_IMAGE_FEATURES=layering RBD_MIRROR_INSTANCES=4 RBD_MIRROR_MODE=snapshot RBD_MIRROR_USE_EXISTING_CLUSTER=1 RBD_MIRROR_USE_RBD_MIRROR=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_stress.sh' ceph version 19.0.0-130-g3d2db44f (3d2db44fa991bc355074aa9b7647f63b77201fef) reef (dev) 1: /lib/x86_64-linux-gnu/libc.so.6(+0x42520) [0x7ff7b924d520] 2: pthread_kill() 3: raise() 4: abort() 5: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0x17b) [0x5560d3674805] 6: (KernelDevice::_aio_thread()+0x9bc) [0x5560d3f5c7fc] 7: ceph-osd(+0x15b40e1) [0x5560d3f3f0e1] 8: /lib/x86_64-linux-gnu/libc.so.6(+0x94b43) [0x7ff7b929fb43] 9: /lib/x86_64-linux-gnu/libc.so.6(+0x126a00) [0x7ff7b9331a00] 2 jobs: ['7486362', '7486337'] suites intersection: ['clients/mirror', 'cluster/{2-node', 'conf/{disable-pool-app}', 'msgr-failures/few', 'openstack}', 'rbd-mirror/four-per-cluster', 'rbd/mirror-thrash/{base/install', 'supported-random-distro$/{ubuntu_latest}'] suites union: ['clients/mirror', 'cluster/{2-node', 'conf/{disable-pool-app}', 'msgr-failures/few', 'objectstore/bluestore-low-osd-mem-target', 'objectstore/bluestore-stupid', 'openstack}', 'policy/none', 'policy/simple', 'rbd-mirror/four-per-cluster', 'rbd/mirror-thrash/{base/install', 'supported-random-distro$/{ubuntu_latest}', 'workloads/rbd-mirror-snapshot-stress-workunit-fast-diff}', 'workloads/rbd-mirror-snapshot-stress-workunit-minimum}'] Failure: Stale jobs detected, aborting. 1 jobs: ['7486219'] suites: ['clusters/{fixed-3', 'conf/{disable-pool-app}', 'data-pool/replicated', 'extra-conf/none', 'min-compat-client/default', 'msgr-failures/few', 'objectstore/bluestore-stupid', 'openstack}', 'rbd/librbd/{cache/writearound', 'supported-random-distro$/{ubuntu_latest}', 'workloads/rbd_fio}'] Failure: ['Failed to connect to the host via ssh: ssh: connect to host smithi059.front.sepia.ceph.com port 22: No route to host'] 1 jobs: ['7486248'] suites: ['conf/{disable-pool-app}', 'objectstore/bluestore-bitmap', 'openstack', 'rbd/singleton-bluestore/{all/issue-20295', 'supported-random-distro$/{centos_latest}}'] Failure: Command failed (workunit test rbd/test_rbd_mirror.sh) on smithi089 with status 134: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=3d2db44fa991bc355074aa9b7647f63b77201fef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 VALGRIND=\'--tool=memcheck --leak-check=full\' adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/test_rbd_mirror.sh' 1 jobs: ['7486363'] suites: ['centos_latest', 'clusters/{fixed-1', 'conf/{disable-pool-app}', 'objectstore/bluestore-comp-lz4', 'openstack}', 'rbd/valgrind/{base/install', 'validator/memcheck', 'workloads/rbd_mirror}'] Failure: Command failed on smithi169 with status 128: 'rm -rf /home/ubuntu/cephtest/qemu_clone.client.0 && git clone https://github.com/ceph/ceph-ci /home/ubuntu/cephtest/qemu_clone.client.0 && cd /home/ubuntu/cephtest/qemu_clone.client.0 && git checkout 3d2db44fa991bc355074aa9b7647f63b77201fef' 1 jobs: ['7486329'] suites: ['clusters/{fixed-3', 'conf/{disable-pool-app}', 'data-pool/none', 'features/defaults', 'msgr-failures/few', 'objectstore/bluestore-bitmap', 'openstack}', 'rbd/encryption/{cache/writethrough', 'supported-random-distro$/{ubuntu_latest}', 'workloads/qemu_xfstests_luks1_luks1}']