Found 33 jobs Found 10 distinct failure reasons Assertion: 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x178) [0x7f3e72e1efd4] ceph version 19.2.3-705-g04cf6dd4 (04cf6dd4be446d06026adad18a5fbf5da1fa8c0e) squid (stable) 1: /lib/x86_64-linux-gnu/libc.so.6(+0x42520) [0x7f3e72775520] 2: pthread_kill() 3: raise() 4: abort() 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x178) [0x7f3e72e1efd4] 6: /usr/lib/ceph/libceph-common.so.2(+0x18512f) [0x7f3e72e1f12f] 7: (AvlAllocator::_add_to_tree(unsigned long, unsigned long)+0x695) [0x55a24de1ef35] 8: (AvlAllocator::init_add_free(unsigned long, unsigned long)+0x9e) [0x55a24de2742e] 9: (BlueStore::_init_alloc(std::map, std::allocator > >*)+0x738) [0x55a24dcf4c48] 10: (BlueStore::_open_db_and_around(bool, bool)+0x3c0) [0x55a24dcfb8c0] 11: (BlueStore::expand_devices(std::ostream&)+0xa4c) [0x55a24dd051cc] 12: main() 13: /lib/x86_64-linux-gnu/libc.so.6(+0x29d90) [0x7f3e7275cd90] 14: __libc_start_main() 15: _start() 1 jobs: ['57882'] suites: ['rados/standalone/{supported-random-distro$/{ubuntu_latest}', 'workloads/osd}'] Failure: "2026-02-19T08:23:32.192735+0000 osd.2 (osd.2) 3 : cluster [WRN] OSD bench result of 101458.980134 IOPS is not within the threshold limit range of 1000.000000 IOPS and 80000.000000 IOPS for osd.2. IOPS capacity is unchanged at 21500.000000 IOPS. The recommendation is to establish the osd's IOPS capacity using other benchmark tools (e.g. Fio) and then override osd_mclock_max_capacity_iops_[hdd|ssd]." in cluster log 5 jobs: ['57853', '57866', '57852', '57872', '57860'] suites intersection: [] suites union: ['0-size-min-size-overrides/2-size-2-min-size', '0-size-min-size-overrides/3-size-2-min-size', '0-start', '1-install/quincy', '1-install/reef', '1-tasks', 'backoff/normal', 'backoff/peering', 'backoff/peering_and_degraded', 'ceph', 'clusters/{openstack', 'd-balancer/crush-compat', 'd-balancer/on', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/fastclose', 'msgr-failures/few', 'msgr-failures/osd-delay', 'overrides/ignorelist_health', 'rados', 'rados/thrash-old-clients/{0-distro$/{centos_9.stream}', 'rados/upgrade/parallel/{0-random-distro$/{centos_9.stream_runc}', 'rados_api', 'rados_loadgenbig', 'rbd_import_export', 'test_rbd_api', 'test_rbd_python}}', 'thrashers/careful', 'thrashers/default', 'thrashers/mapgap', 'thrashers/none', 'thrashosds-health', 'three-plus-one}', 'upgrade-sequence', 'workload/{ec-rados-default', 'workloads/radosbench}', 'workloads/rbd_cls}', 'workloads/test_rbd_api}'] Failure: "2026-02-19T08:33:54.488649+0000 mon.a (mon.0) 319 : cluster [WRN] Health check failed: 2 OSD(s) experiencing slow operations in BlueStore (BLUESTORE_SLOW_OP_ALERT)" in cluster log 6 jobs: ['57870', '57865', '57875', '57862', '57874', '57867'] suites intersection: [] suites union: ['1-start', '2-inject-leak/none', '2-inject-leak/osd', '2-services/rgw', '3-final}', 'centos_latest}', 'ceph', 'clusters/fixed-4', 'd-thrash/none', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/few', 'msgr-failures/none', 'msgr/async-v1only', 'msgr/async-v2only', 'objectstore/bluestore-comp-snappy', 'objectstore/bluestore-comp-zstd', 'rados', 'rados/cephadm/smoke-singlehost/{0-random-distro$/{centos_9.stream}', 'rados/singleton/{all/mon-stretch-pool', 'rados/valgrind-leaks/{1-start', 'rados/verify/{centos_latest', 'supported-random-distro$/{ubuntu_latest}}', 'tasks/mon_recovery', 'tasks/rados_cls_all', 'validater/valgrind}'] Failure: Command failed (workunit test post-file.sh) on trial090 with status 255: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=04cf6dd4be446d06026adad18a5fbf5da1fa8c0e TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/post-file.sh' 2 jobs: ['57857', '57873'] suites intersection: ['rados', 'rados/singleton-nomsgr/{all/ceph-post-file'] suites union: ['mon_election/classic', 'mon_election/connectivity', 'rados', 'rados/singleton-nomsgr/{all/ceph-post-file', 'supported-random-distro$/{centos_latest}}', 'supported-random-distro$/{ubuntu_latest}}'] Crash: Command failed (workunit test cephtool/test_kvstore_tool.sh) on trial105 with status 139: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=04cf6dd4be446d06026adad18a5fbf5da1fa8c0e TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephtool/test_kvstore_tool.sh' ceph version 19.2.3-705-g04cf6dd4 (04cf6dd4be446d06026adad18a5fbf5da1fa8c0e) squid (stable) 1: /lib/x86_64-linux-gnu/libc.so.6(+0x42520) [0x7efe09475520] 2: (RocksDBStore::get_wholespace_iterator(unsigned int)+0xb4) [0x55dfcba38e64] 3: (RocksDBStore::get_iterator(std::__cxx11::basic_string, std::allocator > const&, unsigned int, KeyValueDB::IteratorBounds)+0x519) [0x55dfcba38799] 4: (BlueStore::_close_db()+0x31c) [0x55dfcb88ae6c] 5: (BlueStore::umount()+0x239) [0x55dfcb895439] 6: ceph-kvstore-tool(+0x1f8ea3) [0x55dfcb836ea3] 7: main() 8: /lib/x86_64-linux-gnu/libc.so.6(+0x29d90) [0x7efe0945cd90] 9: __libc_start_main() 10: _start() 2 jobs: ['57871', '57856'] suites intersection: ['rados', 'rados/singleton-nomsgr/{all/ceph-kvstore-tool', 'supported-random-distro$/{ubuntu_latest}}'] suites union: ['mon_election/classic', 'mon_election/connectivity', 'rados', 'rados/singleton-nomsgr/{all/ceph-kvstore-tool', 'supported-random-distro$/{ubuntu_latest}}'] Failure: ["Unexpected failure during module execution: [Errno 28] No space left on device: '/home/teuthworker/.ansible/tmp/ansible-local-1483351a2gkrvor/tmp_34i9e_r'"] 7 jobs: ['57855', '57879', '57883', '57881', '57868', '57880', '57854'] suites intersection: [] suites union: ['0-size-min-size-overrides/2-size-2-min-size', '0-size-min-size-overrides/3-size-2-min-size', '1-install/quincy', '1-install/reef', '2-inject-leak/mon', 'backoff/peering_and_degraded', 'centos_latest}', 'ceph', 'clusters/fixed-4', 'clusters/{openstack', 'd-balancer/crush-compat', 'd-balancer/on', 'd-thrash/none', 'debug/mgr', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/few', 'msgr/async', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-low-osd-mem-target', 'objectstore/bluestore-stupid', 'rados', 'rados/dashboard/{0-single-container-host', 'rados/singleton/{all/backfill-toofull', 'rados/thrash-old-clients/{0-distro$/{centos_9.stream}', 'rados/valgrind-leaks/{1-start', 'rados/verify/{centos_latest', 'random-objectstore$/{bluestore-bitmap}', 'supported-random-distro$/{centos_latest}}', 'tasks/e2e}', 'tasks/rados_api_tests', 'thrashers/careful', 'thrashers/morepggrow', 'thrashosds-health', 'three-plus-one}', 'validater/valgrind}', 'workloads/rbd_cls}', 'workloads/snaps-few-objects}'] Failure: Command failed (workunit test cephtool/test.sh) on trial163 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=04cf6dd4be446d06026adad18a5fbf5da1fa8c0e TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephtool/test.sh' 4 jobs: ['57877', '57869', '57863', '57851'] suites intersection: ['rados', 'rados/singleton-bluestore/{all/cephtool'] suites union: ['mon_election/classic', 'mon_election/connectivity', 'msgr-failures/few', 'msgr-failures/many', 'msgr-failures/none', 'msgr/async', 'msgr/async-v1only', 'msgr/async-v2only', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp-lz4', 'objectstore/bluestore-comp-snappy', 'rados', 'rados/singleton-bluestore/{all/cephtool', 'supported-random-distro$/{centos_latest}}', 'supported-random-distro$/{ubuntu_latest}}'] Failure: Command failed on trial027 with status 1: 'yes | sudo mkfs.xfs -f -i size=2048 -f /dev/vg_nvme/lv_2' 1 jobs: ['57864'] suites: ['debug/mgr', 'mon_election/connectivity', 'rados/dashboard/{0-single-container-host', 'random-objectstore$/{bluestore-stupid}', 'tasks/e2e}'] Failure: Command failed with status 5: 'ansible-playbook -v --extra-vars \'{"ansible_ssh_user": "ubuntu"}\' -i /etc/ansible/hosts --limit trial053.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml' 1 jobs: ['57876'] suites: ['rados/standalone/{supported-random-distro$/{ubuntu_latest}', 'workloads/osd-backfill}'] Dead: b'2026-02-19T14:21:16.809 DEBUG:teuthology.exit:Finished running handlers' 1 jobs: ['57878'] suites: ['0-size-min-size-overrides/3-size-2-min-size', '1-install/reef', 'backoff/peering', 'ceph', 'clusters/{openstack', 'd-balancer/crush-compat', 'mon_election/classic', 'msgr-failures/fastclose', 'rados', 'rados/thrash-old-clients/{0-distro$/{centos_9.stream}', 'thrashers/pggrow', 'thrashosds-health', 'three-plus-one}', 'workloads/radosbench}']