Found 277 jobs Missing teuthology log /home/teuthworker/mnt/teuthology/skanta-2026-02-17_04:03:08-rados-wip-bharath19-testing-2026-02-13-0903-squid-distro-default-trial/51082/teuthology.log Found 12 distinct failure reasons Failure: Command failed on trial016 with status 1: 'yes | sudo mkfs.xfs -f -i size=2048 -f /dev/vg_nvme/lv_2' 2 jobs: ['51117', '51257'] suites intersection: ['debug/mgr', 'rados/dashboard/{0-single-container-host', 'tasks/e2e}'] suites union: ['debug/mgr', 'mon_election/classic', 'mon_election/connectivity', 'rados/dashboard/{0-single-container-host', 'random-objectstore$/{bluestore-bitmap}', 'random-objectstore$/{bluestore-stupid}', 'tasks/e2e}'] Failure: "2026-02-17T05:49:33.338052+0000 mon.a (mon.0) 123 : cluster [WRN] Health check failed: 2 OSD(s) experiencing slow operations in BlueStore (BLUESTORE_SLOW_OP_ALERT)" in cluster log 9 jobs: ['51052', '51184', '51215', '51065', '51232', '51248', '51141', '51124', '51107'] suites intersection: [] suites union: ['1-start', '2-inject-leak/mon', '2-inject-leak/none', '2-inject-leak/osd', '2-services/rgw', '3-final}', 'centos_latest}', 'ceph', 'clusters/fixed-4', 'd-thrash/none', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/few', 'msgr-failures/none', 'msgr/async', 'msgr/async-v1only', 'msgr/async-v2only', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp-snappy', 'objectstore/bluestore-comp-zstd', 'objectstore/bluestore-low-osd-mem-target', 'rados', 'rados/cephadm/smoke-singlehost/{0-random-distro$/{centos_9.stream}', 'rados/singleton/{all/mon-stretch-pool', 'rados/valgrind-leaks/{1-start', 'rados/verify/{centos_latest', 'supported-random-distro$/{ubuntu_latest}}', 'tasks/mon_recovery', 'tasks/rados_api_tests', 'tasks/rados_cls_all', 'validater/valgrind}'] Failure: Command failed (workunit test osd-backfill/osd-backfill-prio.sh) on trial072 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=04cf6dd4be446d06026adad18a5fbf5da1fa8c0e TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/standalone/osd-backfill/osd-backfill-prio.sh' 1 jobs: ['51239'] suites: ['rados/standalone/{supported-random-distro$/{ubuntu_latest}', 'workloads/osd-backfill}'] Failure: "2026-02-17T08:35:10.709770+0000 osd.0 (osd.0) 4 : cluster [WRN] OSD bench result of 90911.482157 IOPS is not within the threshold limit range of 1000.000000 IOPS and 80000.000000 IOPS for osd.0. IOPS capacity is unchanged at 21500.000000 IOPS. The recommendation is to establish the osd's IOPS capacity using other benchmark tools (e.g. Fio) and then override osd_mclock_max_capacity_iops_[hdd|ssd]." in cluster log 10 jobs: ['51129', '51282', '51086', '51190', '51204', '51044', '51051', '51099', '51164', '51243'] suites intersection: [] suites union: ['0-size-min-size-overrides/2-size-2-min-size', '0-size-min-size-overrides/3-size-2-min-size', '0-start', '1-install/quincy', '1-install/reef', '1-tasks', 'backoff/normal', 'backoff/peering', 'backoff/peering_and_degraded', 'ceph', 'clusters/{openstack', 'd-balancer/crush-compat', 'd-balancer/on', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/fastclose', 'msgr-failures/few', 'msgr-failures/osd-delay', 'overrides/ignorelist_health', 'rados', 'rados/singleton-nomsgr/{all/health-warnings', 'rados/thrash-old-clients/{0-distro$/{centos_9.stream}', 'rados/upgrade/parallel/{0-random-distro$/{centos_9.stream_runc}', 'rados/upgrade/parallel/{0-random-distro$/{centos_9.stream}', 'rados_api', 'rados_loadgenbig', 'rbd_import_export', 'supported-random-distro$/{ubuntu_latest}}', 'test_rbd_api', 'test_rbd_python}}', 'thrashers/careful', 'thrashers/default', 'thrashers/mapgap', 'thrashers/morepggrow', 'thrashers/none', 'thrashers/pggrow', 'thrashosds-health', 'three-plus-one}', 'upgrade-sequence', 'workload/{ec-rados-default', 'workloads/radosbench}', 'workloads/rbd_cls}', 'workloads/snaps-few-objects}', 'workloads/test_rbd_api}'] Failure: Command failed (workunit test cephtool/test.sh) on trial195 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=04cf6dd4be446d06026adad18a5fbf5da1fa8c0e TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephtool/test.sh' 4 jobs: ['51040', '51240', '51174', '51109'] suites intersection: ['rados', 'rados/singleton-bluestore/{all/cephtool'] suites union: ['mon_election/classic', 'mon_election/connectivity', 'msgr-failures/few', 'msgr-failures/many', 'msgr-failures/none', 'msgr/async', 'msgr/async-v1only', 'msgr/async-v2only', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp-lz4', 'objectstore/bluestore-comp-snappy', 'rados', 'rados/singleton-bluestore/{all/cephtool', 'supported-random-distro$/{centos_latest}}', 'supported-random-distro$/{ubuntu_latest}}'] Dead: None 2 jobs: ['51082', '51076'] Failure: Command failed (workunit test mon/mon-cluster-log.sh) on trial137 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=04cf6dd4be446d06026adad18a5fbf5da1fa8c0e TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/standalone/mon/mon-cluster-log.sh' 1 jobs: ['51214'] suites: ['rados/standalone/{supported-random-distro$/{ubuntu_latest}', 'workloads/mon}'] Failure: Command failed on trial036 with status 100: 'sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic' 1 jobs: ['51231'] suites: ['clusters/{fixed-4', 'mon_election/classic', 'msgr-failures/osd-delay', 'objectstore/bluestore-hybrid', 'openstack}', 'rados', 'rados/thrash-erasure-code-shec/{ceph', 'recovery-overrides/{more-partial-recovery}', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/careful', 'thrashosds-health', 'workloads/ec-rados-plugin=shec-k=4-m=3-c=2}'] Assertion: 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x178) [0x7fa150d2afd4] ceph version 19.2.3-705-g04cf6dd4 (04cf6dd4be446d06026adad18a5fbf5da1fa8c0e) squid (stable) 1: /lib/x86_64-linux-gnu/libc.so.6(+0x42520) [0x7fa150681520] 2: pthread_kill() 3: raise() 4: abort() 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x178) [0x7fa150d2afd4] 6: /usr/lib/ceph/libceph-common.so.2(+0x18512f) [0x7fa150d2b12f] 7: (AvlAllocator::_add_to_tree(unsigned long, unsigned long)+0x695) [0x55efd358af35] 8: (AvlAllocator::init_add_free(unsigned long, unsigned long)+0x9e) [0x55efd359342e] 9: (BlueStore::_init_alloc(std::map, std::allocator > >*)+0x738) [0x55efd3460c48] 10: (BlueStore::_open_db_and_around(bool, bool)+0x3c0) [0x55efd34678c0] 11: (BlueStore::expand_devices(std::ostream&)+0xa4c) [0x55efd34711cc] 12: main() 13: /lib/x86_64-linux-gnu/libc.so.6(+0x29d90) [0x7fa150668d90] 14: __libc_start_main() 15: _start() 1 jobs: ['51269'] suites: ['rados/standalone/{supported-random-distro$/{ubuntu_latest}', 'workloads/osd}'] Crash: Command failed (workunit test cephtool/test_kvstore_tool.sh) on trial025 with status 139: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=04cf6dd4be446d06026adad18a5fbf5da1fa8c0e TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephtool/test_kvstore_tool.sh' ceph version 19.2.3-705-g04cf6dd4 (04cf6dd4be446d06026adad18a5fbf5da1fa8c0e) squid (stable) 1: /lib/x86_64-linux-gnu/libc.so.6(+0x42520) [0x7fc2db1c8520] 2: (RocksDBStore::get_wholespace_iterator(unsigned int)+0xb4) [0x562376834e64] 3: (RocksDBStore::get_iterator(std::__cxx11::basic_string, std::allocator > const&, unsigned int, KeyValueDB::IteratorBounds)+0x519) [0x562376834799] 4: (BlueStore::_close_db()+0x31c) [0x562376686e6c] 5: (BlueStore::umount()+0x239) [0x562376691439] 6: ceph-kvstore-tool(+0x1f8ea3) [0x562376632ea3] 7: main() 8: /lib/x86_64-linux-gnu/libc.so.6(+0x29d90) [0x7fc2db1afd90] 9: __libc_start_main() 10: _start() 2 jobs: ['51202', '51067'] suites intersection: ['rados', 'rados/singleton-nomsgr/{all/ceph-kvstore-tool', 'supported-random-distro$/{ubuntu_latest}}'] suites union: ['mon_election/classic', 'mon_election/connectivity', 'rados', 'rados/singleton-nomsgr/{all/ceph-kvstore-tool', 'supported-random-distro$/{ubuntu_latest}}'] Failure: 1 jobs: ['51254'] suites: ['mon_election/connectivity', 'msgr-failures/few', 'msgr/async', 'objectstore/bluestore-stupid', 'rados', 'rados/singleton/{all/backfill-toofull', 'supported-random-distro$/{centos_latest}}'] Failure: Command failed (workunit test post-file.sh) on trial173 with status 255: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=04cf6dd4be446d06026adad18a5fbf5da1fa8c0e TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/post-file.sh' 2 jobs: ['51075', '51211'] suites intersection: ['rados', 'rados/singleton-nomsgr/{all/ceph-post-file'] suites union: ['mon_election/classic', 'mon_election/connectivity', 'rados', 'rados/singleton-nomsgr/{all/ceph-post-file', 'supported-random-distro$/{centos_latest}}', 'supported-random-distro$/{ubuntu_latest}}']