Found 146 jobs Found 5 distinct failure reasons Failure: Command crashed: 'sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/ceph.monmap' 49 jobs suites intersection: ['crimson-supported-all-distro/centos_latest', 'crimson_qa_overrides'] suites union: ['0-size-min-size-overrides/3-size-2-min-size', '1-pg-log-overrides/normal_pg_log', '1-pg-log-overrides/short_pg_log', '2-recovery-overrides/{default}', 'all/osd-backfill', 'all/osd-dump-metrics', 'all/osd-pg-splitting', 'clusters/crimson-fixed-1', 'clusters/crimson-fixed-2', 'clusters/{crimson-fixed-1}', 'clusters/{crimson-fixed-2}', 'crimson-rados/basic/{0-crimson_install', 'crimson-rados/perf/{0-crimson_install', 'crimson-rados/rbd/{0-crimson_install', 'crimson-rados/singleton/{0-crimson_install', 'crimson-rados/thrash/{0-crimson_install', 'crimson-supported-all-distro/centos_latest', 'crimson_qa_overrides', 'deploy/ceph', 'objectstore/crimson_bluestore', 'objectstore/seastore/segmented$/{crimson_seastore_segmented_2q}', 'perf_ignorelist', 'rados}', 'settings/optimized', 'tasks/cls}', 'tasks/rados_api_tests}', 'tasks/rados_python}', 'tasks/rbd_api_tests_old_format}', 'tasks/rbd_api_tests}', 'tasks/rbd_cls_tests}', 'tasks/rbd_lock_and_fence}', 'tasks/rbd_python_api_tests_old_format}', 'tasks/rbd_python_api_tests}', 'tasks/readwrite}', 'thrashers/{default}', 'thrashers/{pggrow}', 'thrashosds-health', 'workloads/admin_socket_objecter_requests}', 'workloads/fio_4K_rand_read}', 'workloads/fio_4K_rand_rw}', 'workloads/fio_4M_rand_read}', 'workloads/fio_4M_rand_rw}', 'workloads/fio_4M_rand_write}', 'workloads/pool-snaps-few-objects}', 'workloads/radosbench-high-concurrency}', 'workloads/radosbench_4K_rand_read}', 'workloads/radosbench_4K_seq_read}', 'workloads/radosbench_4M_rand_read}', 'workloads/radosbench_4M_seq_read}', 'workloads/radosbench_4M_write}', 'workloads/radosbench_omap_write}', 'workloads/radosbench}', 'workloads/sample_fio}', 'workloads/sample_radosbench}', 'workloads/small-objects-balanced}', 'workloads/small-objects-localized}', 'workloads/small-objects}', 'workloads/snaps-few-objects-balanced}', 'workloads/snaps-few-objects-localized}', 'workloads/snaps-few-objects}', 'workloads/write_fadvise_dontneed}'] Failure: Exiting scrub checking -- not all pgs scrubbed. 1 jobs: ['61974'] suites: ['0-size-min-size-overrides/3-size-2-min-size', '1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{default}', 'clusters/{crimson-fixed-2}', 'crimson-rados/thrash/{0-crimson_install', 'crimson-supported-all-distro/centos_latest', 'crimson_qa_overrides', 'deploy/ceph', 'objectstore/seastore/segmented$/{crimson_seastore_segmented}', 'thrashers/{pggrow}', 'thrashosds-health', 'workloads/snaps-few-objects-balanced}'] Failure: The WatchDog barked due to Daemon ceph.osd.2 has failed 1 jobs: ['61890'] suites: ['0-size-min-size-overrides/3-size-2-min-size', '1-pg-log-overrides/short_pg_log', '2-recovery-overrides/{default}', 'clusters/{crimson-fixed-2}', 'crimson-rados/thrash/{0-crimson_install', 'crimson-supported-all-distro/centos_latest', 'crimson_qa_overrides', 'deploy/ceph', 'objectstore/seastore/segmented$/{crimson_seastore_segmented}', 'thrashers/{pggrow}', 'thrashosds-health', 'workloads/pool-snaps-few-objects}'] Failure: Command failed (workunit test rbd/test_librbd.sh) on trial124 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=bcdd429b2cc82df735d9dec4bb0cc600cdb61a8f TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_FEATURES=61 CRIMSON_COMPAT=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/test_librbd.sh' 1 jobs: ['61887'] suites: ['clusters/crimson-fixed-1', 'crimson-rados/rbd/{0-crimson_install', 'crimson-supported-all-distro/centos_latest', 'crimson_qa_overrides', 'deploy/ceph', 'objectstore/seastore/segmented$/{crimson_seastore_segmented}', 'tasks/rbd_api_tests}'] Failure: "2026-02-21T00:27:51.333916+0000 mon.a (mon.0) 1243 : cluster [ERR] Health check failed: 1 scrub errors (OSD_SCRUB_ERRORS)" in cluster log 1 jobs: ['61899'] suites: ['clusters/crimson-fixed-2', 'crimson-rados/basic/{0-crimson_install', 'crimson-supported-all-distro/centos_latest', 'crimson_qa_overrides', 'deploy/ceph', 'objectstore/seastore/segmented$/{crimson_seastore_segmented}', 'tasks/rados_api_tests}']