Found 264 jobs Found 28 distinct failure reasons Crash: Command failed on trial103 with status 1: 'sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:squid shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 23775a34-0e3c-11f1-9dc6-d404e6e7d460 -e sha1=138628ee6175b6249df91ec63475221c50a01f37 -- bash -c \'ceph versions | jq -e \'"\'"\'.overall | length == 1\'"\'"\'\'' ceph version 19.2.3-672-g283431c0 (283431c091dafc302f5753f809d0bd1eddce0869) squid (stable) 1: /lib64/libc.so.6(+0x3fc30) [0x7f2552a3fc30] 2: /lib64/libc.so.6(+0x8d03c) [0x7f2552a8d03c] 3: raise() 4: abort() 5: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0x17f) [0x7f2553984524] 6: ceph-fuse(+0xf875b) [0x564effe3975b] 7: ceph-fuse(+0xb642d) [0x564effdf742d] 8: (Finisher::finisher_thread_entry()+0x17d) [0x7f2553a4344d] 9: /lib64/libc.so.6(+0x8b2fa) [0x7f2552a8b2fa] 10: /lib64/libc.so.6(+0x110400) [0x7f2552b10400] 2 jobs: ['59988', '59939'] suites intersection: ['1-ranks/1', '1-volume/{0-create', '2-client/fuse', '3-inline/yes', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_9.stream', 'conf/{client', 'fail_fs/no', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'kernel', 'mds', 'mgr', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'pg_health', 'roles', 'syntax}', 'tasks/{0-from/squid'] suites union: ['1-ranks/1', '1-volume/{0-create', '2-allow_standby_replay/no', '2-allow_standby_replay/yes', '2-client/fuse', '3-inline/yes', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_9.stream', 'conf/{client', 'fail_fs/no', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'kernel', 'mds', 'mgr', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'pg_health', 'roles', 'syntax}', 'tasks/{0-from/squid'] Failure: error during thrashing 10 jobs: ['59978', '60174', '60095', '60154', '60182', '60041', '60071', '60100', '59947', '60006'] suites intersection: ['1-ceph', '2-logrotate', '3-kernel}', 'conf/{client', 'frag', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'osd}', 'overrides/{client-shutdown', 'pg_health', 'session_timeout', 'tasks/{1-thrash/mds', 'thrashosds-health}'] suites union: ['1-ceph', '2-logrotate', '2-workunit/cfuse_workunit_suites_fsstress}}', '2-workunit/cfuse_workunit_suites_pjd}}', '2-workunit/cfuse_workunit_trivial_sync}}', '2-workunit/ffsb}}', '2-workunit/fs/trivial_sync}}', '2-workunit/iozone}}', '2-workunit/suites/iozone}}', '2-workunit/suites/pjd}}', '3-kernel}', 'clusters/1a3s-mds-2c-client', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{centos_latest}', 'distro/{ubuntu_latest}', 'frag', 'fs/thrash/multifs/{begin/{0-install', 'fs/thrash/workloads/{begin/{0-install', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'k-stock}', 'mds', 'mgr', 'mon', 'mount', 'mount/fuse', 'mount/kclient/{mount-syntax/{v1}', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'msgr-failures/none', 'msgr-failures/osd-mds-delay', 'multifs', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{client-shutdown', 'overrides/{distro/stock/{centos_9.stream', 'overrides/{distro/testing/k-testing', 'pg_health', 'prefetch_entire_dirfrags/no', 'prefetch_entire_dirfrags/yes', 'prefetch_oft_dirfrags/no', 'prefetch_oft_dirfrags/yes', 'races', 'ranks/1', 'ranks/3', 'ranks/5', 'session_timeout', 'tasks/{1-thrash/mds', 'thrashosds-health}'] Failure: 7 jobs: ['60130', '60050', '60076', '60159', '60179', '60032', '59997'] suites intersection: ['centos_9.stream', 'conf/{client', 'ignorelist_wrongly_marked_down', 'kernel', 'mds', 'mgr', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'pg_health}'] suites union: ['1-client', '1-mount/mount/kclient/{mount-syntax/{v1}', '2-clients/kclient', '2-upgrade', '3-client-upgrade', '3-compat_client/no}}', '3-compat_client/yes}}', '3-workload/stress_tests/fsstress}}', '4-compat_client', '5-client-sanity}}', 'centos_9.stream', 'clusters/1-mds-2-client-micro', 'clusters/{1-mds-1-client-micro}', 'conf/{client', 'fs/upgrade/featureful_client/old_client/{bluestore-bitmap', 'fs/upgrade/featureful_client/upgraded_client/{bluestore-bitmap', 'fs/upgrade/upgraded_client/{bluestore-bitmap', 'ignorelist_wrongly_marked_down', 'kernel', 'mds', 'mgr', 'mon', 'mount', 'ms-die-on-skipped}}', 'multimds/no', 'multimds/yes', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'pg-warn', 'pg_health}', 'tasks/{0-from/reef', 'tasks/{0-install/reef'] Failure: Command failed (workunit test suites/ffsb.sh) on trial041 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=138628ee6175b6249df91ec63475221c50a01f37 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/ffsb.sh' 1 jobs: ['60042'] suites: ['0-subvolume/{with-namespace-isolated}', '0-use-snaprealm/{global}', '1-cephadm', '1-check-counter', '2-logrotate', '2-scrub/no', '3-kernel}', '3-snaps/yes', '4-flush/no', '5-quiesce/with-quiesce', '6-workunit/suites/ffsb}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'export-check', 'frag', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'ms_mode/secure', 'n/5', 'objectstore-ec/bluestore-comp-ec-root', 'omap_limit/10', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/testing/k-testing', 'pg_health', 'ranks/multi/{balancer/automatic', 'replication/default}', 'session_timeout}', 'standby-replay', 'tasks/{0-fscrypt/{with-fscrypt-dummy}', 'wsync/no}'] Failure: error during quiesce thrashing: Error quiescing set 'ffcd05a1': 110 (ETIMEDOUT) 7 jobs: ['60067', '59977', '60038', '60141', '60155', '59946', '60096'] suites intersection: ['1-cephadm', '1-check-counter', '2-logrotate', '2-scrub/no', '3-kernel}', '5-quiesce/with-quiesce', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'mount', 'ms-die-on-skipped}}', 'omap_limit/10', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/testing/k-testing', 'pg_health', 'session_timeout}', 'standby-replay', 'tasks/{0-fscrypt/{with-fscrypt-dummy}', 'wsync/no}'] suites union: ['0-subvolume/{with-namespace-isolated-and-quota}', '0-subvolume/{with-namespace-isolated}', '0-subvolume/{with-no-extra-options}', '0-subvolume/{with-quota}', '0-use-snaprealm/{global}', '0-use-snaprealm/{subvol}', '1-cephadm', '1-check-counter', '2-logrotate', '2-scrub/no', '3-kernel}', '3-snaps/no', '3-snaps/yes', '4-flush/no', '4-flush/yes', '5-quiesce/with-quiesce', '6-workunit/kernel_untar_build}}', '6-workunit/suites/blogbench}}', '6-workunit/suites/dbench}}', '6-workunit/suites/iogen}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'export-check', 'frag', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v1}', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/crc', 'ms_mode/secure', 'n/3', 'n/5', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'omap_limit/10', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/testing/k-testing', 'pg_health', 'ranks/1', 'ranks/multi/{balancer/automatic', 'ranks/multi/{balancer/random', 'replication/always}', 'replication/default}', 'session_timeout}', 'standby-replay', 'tasks/{0-fscrypt/{with-fscrypt-dummy}', 'wsync/no}'] Failure: reached maximum tries (50) after waiting for 300 seconds 3 jobs: ['60113', '60004', '60170'] suites intersection: ['1-volume/{0-create', '2-client/fuse', '3-inline/no', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_9.stream', 'conf/{client', 'fail_fs/yes', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'kernel', 'mds', 'mgr', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'pg_health', 'roles', 'syntax}', 'tasks/{0-from/squid'] suites union: ['1-ranks/1', '1-ranks/2', '1-volume/{0-create', '2-allow_standby_replay/no', '2-allow_standby_replay/yes', '2-client/fuse', '3-inline/no', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_9.stream', 'conf/{client', 'fail_fs/yes', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'kernel', 'mds', 'mgr', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'pg_health', 'roles', 'syntax}', 'tasks/{0-from/squid'] Assertion: 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x196) [0x7f000fae2f26] ceph version 20.3.0-5338-g138628ee (138628ee6175b6249df91ec63475221c50a01f37) tentacle (dev - None) 1: /lib/x86_64-linux-gnu/libc.so.6(+0x42520) [0x7f000f3e8520] 2: pthread_kill() 3: raise() 4: abort() 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x196) [0x7f000fae2f26] 6: ceph-mds(+0x110ff4) [0x5625395b7ff4] 7: (CInode::pop_and_dirty_projected_inode(std::shared_ptr const&, boost::intrusive_ptr const&)+0x1e5) [0x562539892cb5] 8: (MutationImpl::apply()+0x94) [0x562539726b44] 9: (Server::_unlink_local_finish(boost::intrusive_ptr const&, CDentry*, CDentry*, unsigned long)+0x26c) [0x5625396df0cc] 10: (MDSIOContextBase::complete(int)+0x32c) [0x562539904c7c] 11: (MDSLogContextBase::complete(int)+0x53) [0x562539904dc3] 12: (Finisher::finisher_thread_entry()+0x16d) [0x7f000fbd54dd] 13: /lib/x86_64-linux-gnu/libc.so.6(+0x94ac3) [0x7f000f43aac3] 14: /lib/x86_64-linux-gnu/libc.so.6(+0x1268d0) [0x7f000f4cc8d0] 1 jobs: ['59979'] suites: ['1-ceph', '2-logrotate', '3-kernel}', 'bluestore-bitmap', 'clusters/1-mds-1-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/fscrypt/{begin/{0-install', 'ignorelist_health_more', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'osd', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'pg-warn', 'pg_health}', 'tasks/{1-tests/fscrypt-pjd}}'] Failure: Command failed (workunit test suites/fsx.sh) on trial027 with status 110: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=138628ee6175b6249df91ec63475221c50a01f37 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/fsx.sh' 2 jobs: ['60172', '59937'] suites intersection: ['0-subvolume/{with-namespace-isolated}', '0-use-snaprealm/{global}', '1-cephadm', '1-check-counter', '2-logrotate', '2-scrub/yes', '3-kernel}', '4-flush/no', '5-quiesce/no', '6-workunit/suites/fsx}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'export-check', 'frag', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'mount/fuse', 'n/5', 'objectstore-ec/bluestore-comp', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'pg_health', 'ranks/multi/{balancer/automatic', 'session_timeout}', 'standby-replay', 'tasks/{0-fscrypt/{with-fscrypt-dummy}'] suites union: ['0-subvolume/{with-namespace-isolated}', '0-use-snaprealm/{global}', '1-cephadm', '1-check-counter', '2-logrotate', '2-scrub/yes', '3-kernel}', '3-snaps/no', '3-snaps/yes', '4-flush/no', '5-quiesce/no', '6-workunit/suites/fsx}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'export-check', 'frag', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'mount/fuse', 'n/5', 'objectstore-ec/bluestore-comp', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'pg_health', 'ranks/multi/{balancer/automatic', 'replication/always}', 'replication/default}', 'session_timeout}', 'standby-replay', 'tasks/{0-fscrypt/{with-fscrypt-dummy}'] Failure: Test failure: test_fscrypt_clone (tasks.cephfs.test_fscrypt.TestFSCryptVolumes.test_fscrypt_clone) 2 jobs: ['60191', '60012'] suites intersection: ['1-ceph', '2-logrotate', '3-kernel}', 'bluestore-bitmap', 'clusters/1-mds-1-client', 'conf/{client', 'distro/{centos_latest}', 'fs/fscrypt/{begin/{0-install', 'ignorelist_health_more', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'osd', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'pg_health}', 'tasks/{1-tests/fscrypt-common}}'] suites union: ['1-ceph', '2-logrotate', '3-kernel}', 'bluestore-bitmap', 'clusters/1-mds-1-client', 'conf/{client', 'distro/{centos_latest}', 'fs/fscrypt/{begin/{0-install', 'ignorelist_health_more', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'mount', 'mount/fuse', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'osd', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'pg-warn', 'pg_health}', 'tasks/{1-tests/fscrypt-common}}'] Failure: Test failure: test_async_io_fio (tasks.cephfs.test_nfs.TestNFS.test_async_io_fio) 1 jobs: ['60020'] suites: ['conf/{client', 'fs/nfs/{cluster/{1-node}', 'ignorelist_health', 'mds', 'mgr', 'mon', 'nfs-version/{4.2}', 'osd}', 'overrides/{ignore_mgr_down', 'pg_health}', 'supported-random-distros$/{centos_latest}', 'tasks/nfs}'] Assertion: 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x18e) [0x7f0930585bf5] ceph version 20.3.0-5338-g138628ee (138628ee6175b6249df91ec63475221c50a01f37) tentacle (dev - RelWithDebInfo) 1: /lib64/libc.so.6(+0x3fc30) [0x7f092fa3fc30] 2: /lib64/libc.so.6(+0x8d03c) [0x7f092fa8d03c] 3: raise() 4: abort() 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x18e) [0x7f0930585bf5] 6: ceph-mds() [0x4e109c] 7: (CInode::pop_and_dirty_projected_inode(std::shared_ptr const&, boost::intrusive_ptr const&)+0x214) [0x7d73b4] 8: (MutationImpl::apply()+0x93) [0x65dc73] 9: (Server::_unlink_local_finish(boost::intrusive_ptr const&, CDentry*, CDentry*, unsigned long)+0x2e9) [0x618959] 10: (MDSIOContextBase::complete(int)+0x39c) [0x85312c] 11: (MDSLogContextBase::complete(int)+0x53) [0x8532a3] 12: (Finisher::finisher_thread_entry()+0x16d) [0x7f0930695a7d] 13: /lib64/libc.so.6(+0x8b2fa) [0x7f092fa8b2fa] 14: /lib64/libc.so.6(+0x110400) [0x7f092fb10400] 1 jobs: ['60160'] suites: ['1-ceph', '2-logrotate', '3-kernel}', 'bluestore-bitmap', 'clusters/1-mds-1-client', 'conf/{client', 'distro/{centos_latest}', 'fs/fscrypt/{begin/{0-install', 'ignorelist_health_more', 'ignorelist_wrongly_marked_down', 'k-stock}', 'mds', 'mgr', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'osd', 'osd}', 'overrides/{distro/stock/{centos_9.stream', 'overrides/{ignorelist_health', 'pg-warn', 'pg_health}', 'tasks/{1-tests/fscrypt-pjd}}'] Failure: Test failure: test_progress_event_unique_per_process (tasks.cephfs.test_data_scan.TestDataScan.test_progress_event_unique_per_process) 1 jobs: ['60068'] suites: ['1-ceph', '2-logrotate', '3-kernel}', 'begin/{0-install', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/functional/{0-use-snaprealm/{subvol}', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'mount/fuse', 'no_client_pidfile', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{ignorelist_health', 'pg_health}', 'tasks/data-scan}'] Failure: Command failed on trial165 with status 128: 'rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 && git checkout 138628ee6175b6249df91ec63475221c50a01f37' 3 jobs: ['60124', '60065', '60059'] suites intersection: ['conf/{client', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'osd}'] suites union: ['0-subvolume/{with-no-extra-options}', '0-use-snaprealm/{subvol}', '1-ceph', '1-cephadm', '1-check-counter', '1-ranks/1', '1-volume/{0-create', '2-allow_standby_replay/yes', '2-client/fuse', '2-logrotate', '2-scrub/yes', '3-inline/no', '3-kernel}', '3-snaps/no', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-flush/yes', '4-verify}', '5-quiesce/no', '5-upgrade-with-workload', '6-verify}}', '6-workunit/fs/test_o_trunc}}', 'begin/{0-install', 'bluestore-bitmap', 'centos_9.stream', 'clusters/1-mds-1-client', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'distro/{centos_latest}', 'export-check', 'fail_fs/yes', 'frag', 'fs/fscrypt/{begin/{0-install', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_health_more', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'kernel', 'mds', 'mgr', 'mon', 'mount/fuse', 'n/5', 'objectstore-ec/bluestore-comp', 'omap_limit/10000', 'osd', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{ignorelist_health', 'pg-warn', 'pg_health', 'pg_health}', 'ranks/multi/{balancer/random', 'replication/default}', 'roles', 'session_timeout}', 'standby-replay', 'syntax}', 'tasks/{0-from/squid', 'tasks/{0-fscrypt/{with-fscrypt-dummy}', 'tasks/{1-tests/fscrypt-iozone}}'] Failure: Test failure: test_journal_migration (tasks.cephfs.test_journal_migration.TestJournalMigration.test_journal_migration) 2 jobs: ['60180', '59958'] suites intersection: ['1-ceph', '2-logrotate', '3-kernel}', 'begin/{0-install', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'no_client_pidfile', 'osd}', 'overrides/{ignorelist_health', 'pg_health}', 'tasks/test_journal_migration}'] suites union: ['1-ceph', '2-logrotate', '3-kernel}', 'begin/{0-install', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_latest}', 'distro/{ubuntu_latest}', 'fs/functional/{0-use-snaprealm/{global}', 'fs/functional/{0-use-snaprealm/{subvol}', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'mount', 'mount/fuse', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'no_client_pidfile', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'pg_health}', 'tasks/test_journal_migration}'] Failure: Command failed on trial139 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 1 jobs: ['59996'] suites: ['1-ceph', '2-logrotate', '3-kernel}', 'begin/{0-install', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_latest}', 'fs/functional/{0-use-snaprealm/{global}', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'mount/fuse', 'no_client_pidfile', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{ignorelist_health', 'pg_health}', 'tasks/acls}'] Failure: Command failed on trial160 with status 4: 'rm -f /tmp/kernel.x86_64.rpm && echo kernel-6.19.0_g8dec4b357aab-1.x86_64.rpm | wget -nv -O /tmp/kernel.x86_64.rpm --base=https://2.chacra.ceph.com/r/kernel/testing/8dec4b357aab872d3b63ec979924962a64be83dd/centos/9/flavors/default/x86_64/ --input-file=-' 2 jobs: ['60082', '60081'] suites intersection: ['conf/{client', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'osd}', 'pg_health'] suites union: ['0-subvolume/{with-no-extra-options}', '0-use-snaprealm/{global}', '1-cephadm', '1-check-counter', '1-ranks/2', '1-volume/{0-create', '2-allow_standby_replay/no', '2-client/kclient', '2-logrotate', '2-scrub/no', '3-inline/yes', '3-kernel}', '3-snaps/no', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-flush/yes', '4-verify}', '5-quiesce/with-quiesce', '5-upgrade-with-workload', '6-verify}}', '6-workunit/fs/misc}}', 'begin/{0-install', 'centos_9.stream', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'export-check', 'fail_fs/no', 'frag', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'kernel', 'mds', 'mgr', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'ms_mode/crc', 'n/5', 'objectstore-ec/bluestore-ec-root', 'omap_limit/10', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'pg-warn', 'pg_health', 'ranks/multi/{balancer/random', 'replication/always}', 'roles', 'session_timeout}', 'standby-replay', 'syntax}', 'tasks/{0-from/reef/{v18.2.1}', 'tasks/{0-fscrypt/{with-fscrypt-dummy}', 'wsync/no}'] Failure: Test failure: test_random (tasks.cephfs.test_dump_tree.TestDumpTree.test_random) 1 jobs: ['60008'] suites: ['1-ceph', '2-logrotate', '3-kernel}', 'begin/{0-install', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_latest}', 'fs/functional/{0-use-snaprealm/{subvol}', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'mount/fuse', 'no_client_pidfile', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'pg_health}', 'tasks/asok_dump_tree}'] Dead: b'2026-02-20T16:45:02.813 DEBUG:teuthology.exit:Finished running handlers' 2 jobs: ['60156', '59987'] suites intersection: ['1-ceph', '2-logrotate', '3-kernel}', 'begin/{0-install', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_latest}', 'ignorelist_wrongly_marked_down', 'k-stock}', 'mds', 'mgr', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'no_client_pidfile', 'osd}', 'overrides/{distro/stock/{centos_9.stream', 'overrides/{ignorelist_health', 'pg_health}'] suites union: ['1-ceph', '2-logrotate', '3-kernel}', 'begin/{0-install', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_latest}', 'fs/functional/{0-use-snaprealm/{global}', 'fs/functional/{0-use-snaprealm/{subvol}', 'ignorelist_wrongly_marked_down', 'k-stock}', 'mds', 'mgr', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'no_client_pidfile', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{distro/stock/{centos_9.stream', 'overrides/{ignorelist_health', 'pg_health}', 'tasks/scrub}', 'tasks/xfstests-dev}'] Failure: Command failed (workunit test fs/snaps/snap-rm-diff.sh) on trial068 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=138628ee6175b6249df91ec63475221c50a01f37 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/snaps/snap-rm-diff.sh' 2 jobs: ['60120', '59941'] suites intersection: ['1-ceph', '2-logrotate', '3-kernel}', 'conf/{client', 'frag', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'osd}', 'overrides/{client-shutdown', 'pg_health', 'session_timeout', 'tasks/{1-thrash/mds', 'thrashosds-health}'] suites union: ['1-ceph', '2-logrotate', '2-workunit/cfuse_workunit_snaptests}}', '2-workunit/fs/snaps}}', '3-kernel}', 'clusters/1a3s-mds-2c-client', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{centos_latest}', 'distro/{ubuntu_latest}', 'frag', 'fs/thrash/multifs/{begin/{0-install', 'fs/thrash/workloads/{begin/{0-install', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'mount', 'mount/fuse', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'msgr-failures/none', 'msgr-failures/osd-mds-delay', 'multifs', 'objectstore-ec/bluestore-ec-root', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{client-shutdown', 'overrides/{distro/testing/k-testing', 'pg_health', 'prefetch_entire_dirfrags/no', 'prefetch_oft_dirfrags/yes', 'races', 'ranks/3', 'session_timeout', 'tasks/{1-thrash/mds', 'thrashosds-health}'] Failure: Command failed (workunit test fs/misc/fallocate.sh) on trial004 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=138628ee6175b6249df91ec63475221c50a01f37 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/fallocate.sh' 1 jobs: ['59959'] suites: ['0-subvolume/{with-no-extra-options}', '0-use-snaprealm/{global}', '1-cephadm', '1-check-counter', '2-logrotate', '2-scrub/no', '3-kernel}', '3-snaps/yes', '4-flush/no', '5-quiesce/with-quiesce', '6-workunit/fs/misc}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'export-check', 'frag', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'ms_mode/secure', 'n/3', 'objectstore-ec/bluestore-comp-ec-root', 'omap_limit/10', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/testing/k-testing', 'pg_health', 'ranks/multi/{balancer/random', 'replication/default}', 'session_timeout}', 'standby-replay', 'tasks/{0-fscrypt/{with-fscrypt-dummy}', 'wsync/no}'] Failure: valgrind error: Leak_PossiblyLost operator new[](unsigned long) UnknownInlinedFun UnknownInlinedFun 2 jobs: ['60101', '59994'] suites intersection: ['1-ceph', '2-logrotate', '3-kernel}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'fs/verify/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'mon-debug', 'mount', 'mount/kclient/{k-testing', 'ms-die-on-skipped}', 'osd}', 'overrides/{ignorelist_health', 'pg_health', 'session_timeout}', 'tasks/dbench', 'validater/valgrind}'] suites union: ['1-ceph', '2-logrotate', '3-kernel}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{centos_latest}', 'distro/{ubuntu/{overrides', 'fs/verify/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'mon-debug', 'mount', 'mount/kclient/{k-testing', 'ms-die-on-skipped}', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'osd}', 'overrides/{ignorelist_health', 'pg_health', 'ranks/1', 'ranks/5', 'session_timeout}', 'tasks/dbench', 'ubuntu_latest}}', 'validater/valgrind}'] Failure: Test failure: test_full_fsync (tasks.cephfs.test_full.TestClusterFull.test_full_fsync) 1 jobs: ['60097'] suites: ['1-ceph', '2-logrotate', '3-kernel}', 'begin/{0-install', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_latest}', 'fs/functional/{0-use-snaprealm/{subvol}', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'mount/fuse', 'no_client_pidfile', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{ignorelist_health', 'pg_health}', 'tasks/mds-full}'] Failure: Test failure: test_rename_primary_to_existing_referent_dentry (tasks.cephfs.test_referent.TestReferentInode.test_rename_primary_to_existing_referent_dentry) 1 jobs: ['60144'] suites: ['1-ceph', '2-logrotate', '3-kernel}', 'begin/{0-install', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_latest}', 'fs/functional/{0-use-snaprealm/{subvol}', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'mount/fuse', 'no_client_pidfile', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'pg_health}', 'tasks/referent}'] Failure: "2026-02-20T11:11:11.393803+0000 mon.trial012 (mon.0) 148 : cluster [WRN] Health check failed: Failed to place 1 daemon(s) (CEPHADM_DAEMON_PLACE_FAIL)" in cluster log 1 jobs: ['60186'] suites: ['1-ranks/1', '1-volume/{0-create', '2-allow_standby_replay/no', '2-client/kclient', '3-inline/yes', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_9.stream', 'conf/{client', 'fail_fs/no', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'kernel', 'mds', 'mgr', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'pg_health', 'roles', 'syntax}', 'tasks/{0-from/reef/{reef}'] Failure: Command failed on trial053 with status 1: 'sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v18.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 36942682-0e49-11f1-96c0-d404e6e7d460 -e sha1=138628ee6175b6249df91ec63475221c50a01f37 -- bash -c \'ceph versions | jq -e \'"\'"\'.overall | length == 1\'"\'"\'\'' 2 jobs: ['60136', '60040'] suites intersection: ['1-volume/{0-create', '2-allow_standby_replay/yes', '2-client/kclient', '3-inline/yes', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_9.stream', 'conf/{client', 'fail_fs/no', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'kernel', 'mds', 'mgr', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'pg_health', 'roles', 'syntax}'] suites union: ['1-ranks/1', '1-ranks/2', '1-volume/{0-create', '2-allow_standby_replay/yes', '2-client/kclient', '3-inline/yes', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_9.stream', 'conf/{client', 'fail_fs/no', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'kernel', 'mds', 'mgr', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'pg_health', 'roles', 'syntax}', 'tasks/{0-from/reef/{v18.2.0}', 'tasks/{0-from/reef/{v18.2.1}'] Failure: Command failed (workunit test fs/fscrypt_kclient_fuse.sh) on trial121 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/fscrypt-fuse && cd -- /home/ubuntu/cephtest/mnt.0/fscrypt-fuse && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=138628ee6175b6249df91ec63475221c50a01f37 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 4h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/fscrypt_kclient_fuse.sh fuse' 2 jobs: ['60017', '60148'] suites intersection: ['1-ceph', '2-logrotate', '3-kernel}', 'clusters/1a3s-mds-2c-client', 'conf/{client', 'distro/{ubuntu/{overrides', 'fs/mixed-clients/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'ms-die-on-skipped}', 'osd-asserts', 'osd}', 'overrides/{ignorelist_health', 'pg_health}', 'tasks/fscrypt-kbuild}', 'ubuntu_latest}}'] suites union: ['1-ceph', '2-logrotate', '3-kernel}', 'clusters/1a3s-mds-2c-client', 'conf/{client', 'distro/{ubuntu/{overrides', 'fs/mixed-clients/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'k-stock}', 'kclient-overrides/{distro/stock/{centos_9.stream', 'kclient-overrides/{distro/testing/k-testing', 'mds', 'mgr', 'mon', 'ms-die-on-skipped}', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-ec-root', 'osd-asserts', 'osd}', 'overrides/{ignorelist_health', 'pg_health}', 'tasks/fscrypt-kbuild}', 'ubuntu_latest}}'] Failure: Command failed (workunit test fs/misc/direct_io.py) on trial117 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=138628ee6175b6249df91ec63475221c50a01f37 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/direct_io.py' 1 jobs: ['60138'] suites: ['0-subvolume/{with-quota}', '0-use-snaprealm/{global}', '1-cephadm', '1-check-counter', '2-logrotate', '2-scrub/yes', '3-kernel}', '3-snaps/yes', '4-flush/no', '5-quiesce/no', '6-workunit/fs/misc}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'export-check', 'frag', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'mount/fuse', 'n/3', 'objectstore-ec/bluestore-comp', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'pg_health', 'ranks/multi/{balancer/random', 'replication/default}', 'session_timeout}', 'standby-replay', 'tasks/{0-fscrypt/{with-fscrypt-dummy}'] Failure: Command failed (workunit test suites/fsstress.sh) on trial016 with status 4: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=138628ee6175b6249df91ec63475221c50a01f37 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/fsstress.sh' 1 jobs: ['60109'] suites: ['0-subvolume/{with-quota}', '0-use-snaprealm/{subvol}', '1-cephadm', '1-check-counter', '2-logrotate', '2-scrub/yes', '3-kernel}', '3-snaps/yes', '4-flush/no', '5-quiesce/no', '6-workunit/suites/fsstress}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'export-check', 'frag', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mgr', 'mon', 'mount/fuse', 'n/3', 'objectstore-ec/bluestore-comp', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'pg_health', 'ranks/multi/{balancer/random', 'replication/always}', 'session_timeout}', 'standby-replay', 'tasks/{0-fscrypt/{with-fscrypt-dummy}']