Found 213 jobs Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Saw end of BT but not start Found 18 distinct failure reasons Failure: Command failed on smithi084 with status 5: 'sudo systemctl stop ceph-9826d310-da10-11ee-95c6-87774f69a715@mon.smithi084' 2 jobs: ['7580916', '7581042'] suites intersection: ['conf/{client', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd}', 'overrides/{ignorelist_health'] suites union: ['1-client', '1-ranks/2', '1-volume/{0-create', '2-allow_standby_replay/yes', '2-client/fuse', '2-upgrade', '3-compat_client/quincy}}', '3-inline/no', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_8.stream', 'centos_9.stream', 'clusters/1-mds-2-client-micro', 'conf/{client', 'fail_fs/no', 'fs/upgrade/featureful_client/old_client/{bluestore-bitmap', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'multimds/no', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'pg-warn}', 'roles', 'syntax}', 'tasks/{0-from/reef/{v18.2.1}', 'tasks/{0-octopus'] Failure: No module named 'tasks.fs' 3 jobs: ['7581026', '7581036', '7581058'] suites intersection: ['2-logrotate}', 'conf/{client', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd}'] suites union: ['1-ceph', '1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/no', '3-snaps/no', '4-flush/no', '5-workunit/suites/ffsb}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'export-check', 'frag', 'fs/functional/{begin/{0-install', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/fuse', 'mount/kclient/{mount-syntax/{v1}', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'n/3', 'no_client_pidfile}', 'objectstore-ec/bluestore-comp', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-ec-root', 'omap_limit/10', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'ranks/multi/{balancer/automatic', 'replication/default}', 'session_timeout}', 'standby-replay', 'subvol_versions/create_subvol_version_v1', 'subvol_versions/create_subvol_version_v2', 'tasks/truncate_delay}', 'tasks/workunit/quota}', 'tasks/{0-subvolume/{with-no-extra-options}'] Failure: Test failure: test_replay_beacon_estimated_time (tasks.cephfs.test_failover.TestFailoverBeaconHealth) 2 jobs: ['7581068', '7580965'] suites intersection: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-2c-client', 'conf/{client', 'fs/multifs/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon-debug}', 'osd}', 'overrides/{ignorelist_health', 'tasks/failover}'] suites union: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-2c-client', 'conf/{client', 'distro/{centos_latest}', 'distro/{ubuntu_latest}', 'fs/multifs/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'k-stock}', 'mds', 'mon', 'mon-debug}', 'mount', 'mount/fuse', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'osd}', 'overrides/{distro/stock/{centos_9.stream', 'overrides/{ignorelist_health', 'tasks/failover}'] Failure: Command failed (workunit test suites/fsx.sh) on smithi118 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=717ce3cce8d0166e4e577faa8004e6f9cb4128c0 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/fsx.sh' 2 jobs: ['7580943', '7581054'] suites intersection: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/yes', '3-snaps/yes', '5-workunit/suites/fsx}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'export-check', 'frag', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'ranks/multi/{balancer/random', 'session_timeout}', 'standby-replay'] suites union: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/yes', '3-snaps/yes', '4-flush/no', '4-flush/yes', '5-workunit/suites/fsx}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'export-check', 'frag', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'k-stock}', 'mds', 'mon', 'mount', 'mount/fuse', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/secure', 'n/3', 'n/5', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-comp-ec-root', 'omap_limit/10', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/stock/{centos_9.stream', 'ranks/multi/{balancer/random', 'replication/always}', 'replication/default}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-namespace-isolated-and-quota}', 'tasks/{0-subvolume/{with-no-extra-options}', 'wsync/no}'] Failure: Command failed (workunit test suites/dbench.sh) on smithi193 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=717ce3cce8d0166e4e577faa8004e6f9cb4128c0 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 CEPH_MNT=/home/ubuntu/cephtest/mnt.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.1/qa/workunits/suites/dbench.sh' 2 jobs: ['7580921', '7581010'] suites intersection: [] suites union: ['1-cephadm', '1-check-counter', '1-mount', '2-logrotate}', '2-scrub/yes', '2-workload/dbench', '3-snaps/yes', '4-flush/yes', '5-workunit/suites/dbench}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'distro/single-container-host}', 'export-check', 'frag', 'fs/cephadm/multivolume/{0-start', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'k-stock}', 'mds', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/secure', 'n/5', 'objectstore-ec/bluestore-comp-ec-root', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/stock/{centos_9.stream', 'ranks/multi/{balancer/random', 'replication/always}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-quota}', 'wsync/no}'] Failure: reached maximum tries (51) after waiting for 300 seconds 4 jobs: ['7581129', '7580988', '7581048', '7580881'] suites intersection: ['1-volume/{0-create', '2-client/fuse', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_9.stream', 'conf/{client', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}'] suites union: ['1-ranks/1', '1-ranks/2', '1-volume/{0-create', '2-allow_standby_replay/no', '2-allow_standby_replay/yes', '2-client/fuse', '3-inline/no', '3-inline/yes', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_9.stream', 'conf/{client', 'fail_fs/no', 'fail_fs/yes', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}', 'tasks/{0-from/quincy', 'tasks/{0-from/reef/{reef}'] Crash: Test failure: test_object_deletion (tasks.cephfs.test_damage.TestDamage) ceph version 19.0.0-1845-g717ce3cc (717ce3cce8d0166e4e577faa8004e6f9cb4128c0) squid (dev) 1: /lib64/libc.so.6(+0x54db0) [0x7f935a254db0] 2: /lib64/libc.so.6(+0xa154c) [0x7f935a2a154c] 3: raise() 4: abort() 5: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0x190) [0x7f935ab815d4] 6: (MDSRank::abort(std::basic_string_view >)+0x61) [0x55714300e4f1] 7: (CDentry::check_corruption(bool)+0x48d) [0x55714321dd7d] 8: ceph-mds(+0x37bfec) [0x55714322ffec] 9: (CDir::_omap_commit(int)+0x93a) [0x55714323754a] 10: (CDir::commit(unsigned long, MDSContext*, bool, int)+0x265) [0x55714322e845] 11: (LogSegment::try_to_expire(MDSRank*, C_GatherBuilderBase >&, int)+0xa61) [0x557143053991] 12: (MDLog::try_expire(LogSegment*, int)+0x62) [0x5571432b80f2] 13: (MDLog::trim_all()+0x708) [0x5571432bb0c8] 14: ceph-mds(+0x18e0c1) [0x5571430420c1] 15: ceph-mds(+0x133dcd) [0x557142fe7dcd] 16: (MDSContext::complete(int)+0x5c) [0x5571432b0bdc] 17: ceph-mds(+0x134072) [0x557142fe8072] 18: (MDSContext::complete(int)+0x5c) [0x5571432b0bdc] 19: (MDSIOContextBase::complete(int)+0x31c) [0x5571432b102c] 20: (Finisher::finisher_thread_entry()+0x175) [0x7f935ac2c9a5] 21: /lib64/libc.so.6(+0x9f802) [0x7f935a29f802] 22: /lib64/libc.so.6(+0x3f450) [0x7f935a23f450] 1 jobs: ['7580900'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'k-stock}', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{distro/stock/{centos_9.stream', 'overrides/{ignorelist_health', 'subvol_versions/create_subvol_version_v2', 'tasks/damage}'] Failure: Test failure: test_reading_conf (tasks.cephfs.test_cephfs_shell.TestShellOpts) 1 jobs: ['7580930'] suites: ['1-ceph', '2-logrotate}', 'clusters/1-mds-1-client-coloc', 'conf/{client', 'distro/centos_latest', 'fs/shell/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'tasks/cephfs-shell}'] Failure: error during scrub thrashing: rank damage found: {'backtrace'} 1 jobs: ['7580985'] suites: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/yes', '3-snaps/yes', '4-flush/yes', '5-workunit/fs/misc}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'export-check', 'frag', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'k-stock}', 'mds', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'ms_mode/secure', 'n/5', 'objectstore-ec/bluestore-comp-ec-root', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/stock/{centos_9.stream', 'ranks/multi/{balancer/automatic', 'replication/always}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-quota}', 'wsync/no}'] Failure: "2024-03-04T10:30:00.000147+0000 mon.a (mon.0) 1281 : cluster 3 [WRN] MDS_SLOW_METADATA_IO: 3 MDSs report slow metadata IOs" in cluster log 2 jobs: ['7580913', '7580868'] suites intersection: ['conf/{client', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd}'] suites union: ['1-cephadm', '1-check-counter', '1-ranks/2', '1-volume/{0-create', '2-allow_standby_replay/no', '2-client/kclient', '2-logrotate}', '2-scrub/no', '3-inline/yes', '3-snaps/no', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-flush/yes', '4-verify}', '5-upgrade-with-workload', '5-workunit/suites/ffsb}}', '6-verify}}', 'begin/{0-install', 'centos_9.stream', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'export-check', 'fail_fs/yes', 'frag', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'k-stock}', 'mds', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'ms_mode/crc', 'n/5', 'objectstore-ec/bluestore-ec-root', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/stock/{centos_9.stream', 'overrides/{ignorelist_health', 'pg-warn', 'ranks/multi/{balancer/random', 'replication/default}', 'roles', 'session_timeout}', 'standby-replay', 'syntax}', 'tasks/{0-from/quincy', 'tasks/{0-subvolume/{with-quota}', 'wsync/no}'] Dead: b'2024-03-05T00:01:04.298 DEBUG:teuthology.exit:Finished running handlers' ceph version 19.0.0-1845-g717ce3cc (717ce3cce8d0166e4e577faa8004e6f9cb4128c0) squid (dev) 1: /lib64/libc.so.6(+0x54db0) [0x7f21a6c54db0] 2: /lib64/libc.so.6(+0xa154c) [0x7f21a6ca154c] 3: raise() 4: abort() 5: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0x187) [0x555f68102512] 6: (KernelDevice::_aio_thread()+0x11e0) [0x555f68ad1f60] 7: ceph-osd(+0xdc50d1) [0x555f68ad20d1] 8: /lib64/libc.so.6(+0x9f802) [0x7f21a6c9f802] 9: /lib64/libc.so.6(+0x3f450) [0x7f21a6c3f450] 1 jobs: ['7581052'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'k-stock}', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{distro/stock/{centos_9.stream', 'overrides/{ignorelist_health', 'subvol_versions/create_subvol_version_v2', 'tasks/xfstests-dev}'] Failure: Command failed on smithi133 with status 100: 'sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=19.0.0-1845-g717ce3cc-1jammy ceph-mds=19.0.0-1845-g717ce3cc-1jammy ceph-common=19.0.0-1845-g717ce3cc-1jammy ceph-fuse=19.0.0-1845-g717ce3cc-1jammy ceph-test=19.0.0-1845-g717ce3cc-1jammy radosgw=19.0.0-1845-g717ce3cc-1jammy python-ceph=19.0.0-1845-g717ce3cc-1jammy libcephfs1=19.0.0-1845-g717ce3cc-1jammy libcephfs-java=19.0.0-1845-g717ce3cc-1jammy libcephfs-jni=19.0.0-1845-g717ce3cc-1jammy librados2=19.0.0-1845-g717ce3cc-1jammy librbd1=19.0.0-1845-g717ce3cc-1jammy rbd-fuse=19.0.0-1845-g717ce3cc-1jammy python3-cephfs=19.0.0-1845-g717ce3cc-1jammy cephfs-shell=19.0.0-1845-g717ce3cc-1jammy cephfs-top=19.0.0-1845-g717ce3cc-1jammy cephfs-mirror=19.0.0-1845-g717ce3cc-1jammy' 1 jobs: ['7581044'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'subvol_versions/create_subvol_version_v1', 'tasks/workunit/dir-max-entries}'] Failure: Test failure: test_cephfs_mirror_cancel_mirroring_and_readd (tasks.cephfs.test_mirroring.TestMirroring) 1 jobs: ['7580933'] suites: ['1-ceph', '2-logrotate}', 'centos_latest', 'clients/mirror', 'cluster/1-node', 'debug', 'fs/valgrind/{begin/{0-install', 'mirror/{cephfs-mirror/one-per-cluster', 'mount/fuse', 'overrides/ignorelist_health', 'tasks/mirror}}'] Failure: Test failure: test_fscrypt_dummy_encryption_with_quick_group (tasks.cephfs.test_fscrypt.TestFscrypt) 1 jobs: ['7581131'] suites: ['1-ceph', '1-tests/fscrypt-common}}', '2-logrotate}', 'bluestore-bitmap', 'clusters/1-mds-1-client', 'conf/{client', 'distro/{centos_latest}', 'fs/fscrypt/{begin/{0-install', 'ignorelist_health_more', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'osd', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'pg-warn}', 'tasks/{0-client'] Failure: Test failure: test_cd_with_args (tasks.cephfs.test_cephfs_shell.TestCD) 1 jobs: ['7581072'] suites: ['1-ceph', '2-logrotate}', 'clusters/1-mds-1-client-coloc', 'conf/{client', 'distro/ubuntu_latest', 'fs/shell/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'tasks/cephfs-shell}'] Failure: valgrind error: Leak_PossiblyLost calloc __trans_list_add 2 jobs: ['7581034', '7580885'] suites intersection: ['1-ceph', '2-logrotate}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{ubuntu/{overrides', 'fs/verify/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon-debug', 'osd}', 'overrides/{ignorelist_health', 'ranks/3', 'session_timeout}', 'tasks/dbench', 'ubuntu_latest}}', 'validater/valgrind}'] suites union: ['1-ceph', '2-logrotate}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{ubuntu/{overrides', 'fs/verify/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon-debug', 'mount', 'mount/fuse', 'mount/kclient/{k-testing', 'ms-die-on-skipped}', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-comp-ec-root', 'osd}', 'overrides/{ignorelist_health', 'ranks/3', 'session_timeout}', 'tasks/dbench', 'ubuntu_latest}}', 'validater/valgrind}'] Timeout 3h running clone.client.0/qa/workunits/suites/fsstress.sh 1 jobs: ['7580951'] suites: ['1-ranks/1', '1-volume/{0-create', '2-allow_standby_replay/no', '2-client/kclient', '3-inline/yes', '3-upgrade-mgr-staggered', '4-config-upgrade/{fail_fs}', '4-verify}', '5-upgrade-with-workload', '6-verify}}', 'centos_9.stream', 'conf/{client', 'fail_fs/yes', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'ignorelist_upgrade', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn', 'roles', 'syntax}', 'tasks/{0-from/quincy'] Timeout 3h running clone.client.0/qa/workunits/fs/misc/multiple_rsync.sh 1 jobs: ['7580882'] suites: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/yes', '3-snaps/yes', '4-flush/no', '5-workunit/fs/misc}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'export-check', 'frag', 'fs/workload/{0-centos_9.stream', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'n/5', 'objectstore-ec/bluestore-comp', 'omap_limit/10', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'ranks/multi/{balancer/automatic', 'replication/default}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-quota}']