Found 78 jobs Misunderstood line: Missing teuthology log /home/teuthworker/archive/pdonnell-2023-03-10_16:07:47-fs-wip-pdonnell-testing-20230308.150904-distro-default-smithi/7201921/teuthology.log Misunderstood line: Found 17 distinct failure reasons Failure: Command failed (workunit test suites/pjd.sh) on smithi084 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=aa4319999aed3b25ade5f5d8ada5e531ce3d14cd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/pjd.sh' 9 jobs: ['7201884', '7201919', '7201932', '7201860', '7201871', '7201902', '7201879', '7201896', '7201911'] suites intersection: ['1-cephadm', '1-check-counter', '2-logrotate}', '5-workunit/suites/pjd}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'session_timeout}', 'standby-replay'] suites union: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/no', '2-scrub/yes', '3-snaps/no', '3-snaps/yes', '4-flush/no', '4-flush/yes', '5-workunit/suites/pjd}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/fuse', 'mount/kclient/{base/{mount-syntax/{v1}', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/crc', 'ms_mode/legacy', 'ms_mode/secure', 'n/3', 'n/5', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'omap_limit/10', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/stock/{k-stock', 'overrides/{distro/testing/k-testing', 'ranks/1', 'ranks/multi/{export-check', 'replication/always}', 'replication/default}', 'rhel_8}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-namespace-isolated-and-quota}', 'tasks/{0-subvolume/{with-namespace-isolated}', 'tasks/{0-subvolume/{with-no-extra-options}', 'tasks/{0-subvolume/{with-quota}', 'wsync/no}', 'wsync/yes}'] Failure: error during scrub thrashing: rank damage found: {'backtrace'} 13 jobs: ['7201904', '7201865', '7201886', '7201855', '7201872', '7201889', '7201920', '7201905', '7201874', '7201873', '7201870', '7201892', '7201929'] suites intersection: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/yes', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'ranks/multi/{export-check', 'session_timeout}', 'standby-replay'] suites union: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/yes', '3-snaps/no', '3-snaps/yes', '4-flush/no', '4-flush/yes', '5-workunit/fs/misc}}', '5-workunit/fs/norstats}}', '5-workunit/kernel_untar_build}}', '5-workunit/postgres}}', '5-workunit/suites/blogbench}}', '5-workunit/suites/ffsb}}', '5-workunit/suites/fsstress}}', '5-workunit/suites/iozone}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/fuse', 'mount/kclient/{base/{mount-syntax/{v1}', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/crc', 'ms_mode/legacy', 'ms_mode/secure', 'n/3', 'n/5', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'omap_limit/10', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/stock/{k-stock', 'overrides/{distro/testing/k-testing', 'ranks/multi/{export-check', 'replication/always}', 'replication/default}', 'rhel_8}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{no-subvolume}', 'tasks/{0-subvolume/{with-namespace-isolated-and-quota}', 'tasks/{0-subvolume/{with-namespace-isolated}', 'tasks/{0-subvolume/{with-no-extra-options}', 'tasks/{0-subvolume/{with-quota}', 'wsync/no}', 'wsync/yes}'] Dead: b'2023-03-12T03:50:51.177 DEBUG:teuthology.exit:Finished running handlers' 7 jobs: ['7201857', '7201880', '7201898', '7201913', '7201866', '7201915', '7201887'] suites intersection: ['1-cephadm', '1-check-counter', '2-logrotate}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'ranks/multi/{export-check', 'session_timeout}', 'standby-replay'] suites union: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/no', '2-scrub/yes', '3-snaps/no', '3-snaps/yes', '4-flush/no', '4-flush/yes', '5-workunit/fs/misc}}', '5-workunit/postgres}}', '5-workunit/suites/fsstress}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/fuse', 'mount/kclient/{base/{mount-syntax/{v1}', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/crc', 'ms_mode/legacy', 'ms_mode/secure', 'n/3', 'n/5', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'omap_limit/10', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/stock/{k-stock', 'overrides/{distro/testing/k-testing', 'ranks/multi/{export-check', 'replication/always}', 'replication/default}', 'rhel_8}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{no-subvolume}', 'tasks/{0-subvolume/{with-namespace-isolated-and-quota}', 'tasks/{0-subvolume/{with-namespace-isolated}', 'tasks/{0-subvolume/{with-no-extra-options}', 'tasks/{0-subvolume/{with-quota}', 'wsync/no}', 'wsync/yes}'] Valgrind: mon (InvalidRead, Leak_PossiblyLost, Leak_StillReachable) 2 jobs: ['7201864', '7201895'] suites intersection: ['1-ceph', '2-logrotate}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{ubuntu/{latest', 'fs/verify/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon-debug', 'mount/fuse', 'osd}', 'overrides/{ignorelist_health', 'overrides}}', 'session_timeout}', 'tasks/dbench', 'validater/valgrind}'] suites union: ['1-ceph', '2-logrotate}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{ubuntu/{latest', 'fs/verify/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon-debug', 'mount/fuse', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'osd}', 'overrides/{ignorelist_health', 'overrides}}', 'ranks/1', 'ranks/5', 'session_timeout}', 'tasks/dbench', 'validater/valgrind}'] Failure: Command failed (workunit test suites/dbench.sh) on smithi003 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=aa4319999aed3b25ade5f5d8ada5e531ce3d14cd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/dbench.sh' 2 jobs: ['7201867', '7201927'] suites intersection: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/yes', '3-snaps/yes', '4-flush/yes', '5-workunit/suites/dbench}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'ms-die-on-skipped}}', 'ms_mode/secure', 'objectstore-ec/bluestore-ec-root', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/testing/k-testing', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-namespace-isolated-and-quota}', 'wsync/no}'] suites union: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/yes', '3-snaps/yes', '4-flush/yes', '5-workunit/suites/dbench}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v1}', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/secure', 'n/3', 'objectstore-ec/bluestore-ec-root', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/testing/k-testing', 'ranks/1', 'ranks/multi/{export-check', 'replication/default}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-namespace-isolated-and-quota}', 'wsync/no}'] Failure: Test failure: test_open_ino_errors (tasks.cephfs.test_damage.TestDamage) 1 jobs: ['7201899'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{ignorelist_health', 'tasks/damage}'] Dead: None 5 jobs: ['7201921', '7201876', '7201924', '7201858', '7201897'] Failure: Test failure: test_extra_data_pool_rebuild_few_files (tasks.cephfs.test_data_scan.TestDataScan) 1 jobs: ['7201900'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{ignorelist_health', 'rhel_8}', 'tasks/data-scan}'] Failure: Command failed (workunit test suites/ffsb.sh) on smithi088 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=aa4319999aed3b25ade5f5d8ada5e531ce3d14cd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/ffsb.sh' 2 jobs: ['7201906', '7201891'] suites intersection: ['1-cephadm', '1-check-counter', '2-logrotate}', '4-flush/yes', '5-workunit/suites/ffsb}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'ms-die-on-skipped}}', 'n/3', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/testing/k-testing', 'ranks/multi/{export-check', 'replication/always}', 'session_timeout}', 'standby-replay', 'wsync/no}'] suites union: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/no', '2-scrub/yes', '3-snaps/no', '3-snaps/yes', '4-flush/yes', '5-workunit/suites/ffsb}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v1}', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/crc', 'ms_mode/secure', 'n/3', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/testing/k-testing', 'ranks/multi/{export-check', 'replication/always}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{no-subvolume}', 'tasks/{0-subvolume/{with-quota}', 'wsync/no}'] Timeout 6h running clone.client.0/qa/workunits/suites/fsstress.sh 2 jobs: ['7201869', '7201878'] suites intersection: ['2-logrotate}', 'conf/{client', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'osd}', 'session_timeout}'] suites union: ['1-ceph', '1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/no', '3-snaps/no', '4-flush/no', '5-workunit/suites/fsstress}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{rhel_8}', 'frag', 'fs/verify/{begin/{0-install', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon-debug', 'mount', 'mount/kclient/{base/{mount-syntax/{v2}', 'mount/kclient/{k-testing', 'ms-die-on-skipped}', 'ms-die-on-skipped}}', 'ms_mode/legacy', 'n/5', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-ec-root', 'omap_limit/10', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/stock/{k-stock', 'overrides/{ignorelist_health', 'ranks/3', 'ranks/multi/{export-check', 'replication/always}', 'rhel_8}', 'session_timeout}', 'standby-replay', 'tasks/fsstress', 'tasks/{0-subvolume/{with-namespace-isolated-and-quota}', 'validater/lockdep}', 'wsync/yes}'] Failure: Command failed (workunit test fs/quota/quota.sh) on smithi196 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=aa4319999aed3b25ade5f5d8ada5e531ce3d14cd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 CEPH_MNT=/home/ubuntu/cephtest/mnt.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.1/qa/workunits/fs/quota/quota.sh' 1 jobs: ['7201877'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{ignorelist_health', 'rhel_8}', 'tasks/workunit/quota}'] Dead: b'2023-03-12T05:26:32.274 INFO:journalctl@ceph.mon.c.smithi162.stdout:Mar 12 05:26:31 smithi162 ceph-mon[125139]: pgmap v22836: 129 pgs: 129 active+clean; 8.8 GiB data, 5.8 GiB used, 1.0 TiB / 1.0 TiB avail; 510 KiB/s rd, 513 KiB/s wr, 11 op/s' 1 jobs: ['7201926'] suites: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/yes', '3-snaps/yes', '4-flush/yes', '5-workunit/postgres}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/crc', 'n/5', 'objectstore-ec/bluestore-comp-ec-root', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/testing/k-testing', 'ranks/multi/{export-check', 'replication/default}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-quota}', 'wsync/no}'] Failure: reached maximum tries (50) after waiting for 300 seconds 1 jobs: ['7201861'] suites: ['1-ceph', '2-logrotate}', 'cephfs-mirror/three-per-cluster', 'clients/{mirror}', 'cluster/{1-node}', 'fs/mirror-ha/{begin/{0-install', 'objectstore/bluestore-bitmap', 'overrides/{whitelist_health}', 'supported-random-distro$/{centos_8}', 'workloads/cephfs-mirror-ha-workunit}'] Failure: Test failure: test_client_min_caps_working_set (tasks.cephfs.test_client_limits.TestClientLimits) 1 jobs: ['7201893'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{rhel_8}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'tasks/client-limits}'] Failure: Test failure: test_rebuild_simple (tasks.cephfs.test_recovery_pool.TestRecoveryPool) 1 jobs: ['7201883'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_8}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{ignorelist_health', 'rhel_8}', 'tasks/alternate-pool}'] Crash: Test failure: test_inject_to_empty (tasks.cephfs.test_journal_repair.TestJournalRepair) ceph version 18.0.0-2872-ge0ab30a2 (e0ab30a2acead6db62b144ba6b6aeb23635adae7) reef (dev) 1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7f2652ee9420] 2: gsignal() 3: abort() 4: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x182) [0x7f265231e2af] 5: /usr/lib/ceph/libceph-common.so.2(+0x31e411) [0x7f265231e411] 6: (CDir::log_mark_dirty()+0xfa) [0x5634fa06557a] 7: (CDir::_omap_fetched(ceph::buffer::v15_2_0::list&, std::map, std::allocator >, ceph::buffer::v15_2_0::list, std::less, std::allocator > >, std::allocator, std::allocator > const, ceph::buffer::v15_2_0::list> > >&, bool, std::set, std::allocator >, std::less, std::allocator > >, std::allocator, std::allocator > > > const&, int)+0x1104) [0x5634fa066694] 8: (C_IO_Dir_OMAP_Fetched::finish(int)+0xbb) [0x5634fa07d14b] 9: (MDSContext::complete(int)+0x5f) [0x5634fa11d81f] 10: (MDSIOContextBase::complete(int)+0x514) [0x5634fa11df84] 11: (Finisher::finisher_thread_entry()+0x18d) [0x7f26523c490d] 12: /lib/x86_64-linux-gnu/libpthread.so.0(+0x8609) [0x7f2652edd609] 13: clone() 1 jobs: ['7201908'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'tasks/journal-repair}'] Failure: Command failed (workunit test fs/snaps/snaptest-multiple-capsnaps.sh) on smithi123 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=aa4319999aed3b25ade5f5d8ada5e531ce3d14cd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/snaps/snaptest-multiple-capsnaps.sh' 1 jobs: ['7201903'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-1c-client', 'conf/{client', 'distro/{rhel_8}', 'fs/snaps/{begin/{0-install', 'ignorelist_wrongly_marked_down}', 'mds', 'mon', 'mount/fuse', 'objectstore-ec/bluestore-ec-root', 'osd}', 'overrides/{ignorelist_health', 'tasks/workunit/snaps}']