Found 27 jobs Misunderstood line: Misunderstood line: Found 13 distinct failure reasons Failure: Command failed on smithi170 with status 2: 'TESTDIR=/home/ubuntu/cephtest bash -s' 4 jobs: ['7341029', '7341041', '7341048', '7341036'] suites intersection: ['1-ceph', '2-logrotate}', 'conf/{client', 'distros/ubuntu_latest', 'fs/multiclient/{begin/{0-install', 'ignorelist_wrongly_marked_down}', 'mds', 'mon', 'mount/fuse', 'osd}', 'overrides/{ignorelist_health'] suites union: ['1-ceph', '2-logrotate}', 'clusters/1-mds-2-client', 'clusters/1-mds-3-client', 'conf/{client', 'distros/ubuntu_latest', 'fs/multiclient/{begin/{0-install', 'ignorelist_wrongly_marked_down}', 'mds', 'mon', 'mount/fuse', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'osd}', 'overrides/{ignorelist_health', 'tasks/ior-shared-file}', 'tasks/mdtest}'] Dead: b'2023-07-16T23:06:26.328 DEBUG:teuthology.orchestra.run.smithi160:> sudo logrotate /etc/logrotate.d/ceph-test.conf' ceph version 18.0.0-4725-gddde6a3b (ddde6a3b6f51074e32a7fa89805bbbb065d047c2) reef (dev) 1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7f14229d6420] 2: gsignal() 3: abort() 4: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x182) [0x7f1421f1e2b1] 5: /usr/lib/ceph/libceph-common.so.2(+0x31e413) [0x7f1421f1e413] 6: (SimpleLock::put_xlock()+0x332) [0x561a9d898b02] 7: (Locker::xlock_finish(std::_Rb_tree_const_iterator const&, MutationImpl*, bool*)+0x8c) [0x561a9d8826ec] 8: (Locker::_drop_locks(MutationImpl*, std::set, std::allocator >*, bool)+0x3f0) [0x561a9d882f50] 9: (Locker::drop_non_rdlocks(MutationImpl*, std::set, std::allocator >*)+0x5f) [0x561a9d8839af] 10: (Server::reply_client_request(boost::intrusive_ptr&, boost::intrusive_ptr const&)+0x48d) [0x561a9d6ee29d] 11: (Server::respond_to_request(boost::intrusive_ptr&, int)+0x350) [0x561a9d6ef0e0] 12: (Server::_unlink_local_finish(boost::intrusive_ptr&, CDentry*, CDentry*, unsigned long)+0x2e1) [0x561a9d6f1941] 13: (MDSContext::complete(int)+0x5f) [0x561a9d9c333f] 14: (MDSIOContextBase::complete(int)+0x514) [0x561a9d9c3aa4] 15: (MDSLogContextBase::complete(int)+0x5b) [0x561a9d9c3e3b] 16: (Finisher::finisher_thread_entry()+0x18d) [0x7f1421fc4e1d] 17: /lib/x86_64-linux-gnu/libpthread.so.0(+0x8609) [0x7f14229ca609] 18: clone() 2 jobs: ['7341052', '7341047'] suites intersection: ['1-ceph', '2-logrotate}', 'conf/{client', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd}', 'overrides/{ignorelist_health'] suites union: ['1-ceph', '1-tests/fscrypt-common}}', '2-logrotate}', 'bluestore-bitmap', 'clusters/1-mds-1-client', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_20.04}', 'distro/{ubuntu_latest}', 'fs/fscrypt/{begin/{0-install', 'fs/functional/{begin/{0-install', 'ignorelist_health_more', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/fuse', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'pg-warn}', 'tasks/xfstests-dev}', 'tasks/{0-client'] Failure: Command failed (workunit test fs/quota/quota.sh) on smithi138 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=ddde6a3b6f51074e32a7fa89805bbbb065d047c2 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 CEPH_MNT=/home/ubuntu/cephtest/mnt.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.1/qa/workunits/fs/quota/quota.sh' 2 jobs: ['7341054', '7341032'] suites intersection: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'tasks/workunit/quota}'] suites union: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_8}', 'distro/{ubuntu_20.04}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'rhel_8}', 'tasks/workunit/quota}'] Failure: Test failure: test_generic (tasks.cephfs.tests_from_xfstests_dev.TestXFSTestsDev) 1 jobs: ['7341031'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_8}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{ignorelist_health', 'rhel_8}', 'tasks/xfstests-dev}'] Failure: Test failure: test_disable_enable_human_readable_quota_values (tasks.cephfs.test_quota.TestQuota) 1 jobs: ['7341045'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{rhel_8}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{ignorelist_health', 'rhel_8}', 'tasks/quota}'] Failure: Command failed (workunit test fs/test_python.sh) on smithi086 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=ddde6a3b6f51074e32a7fa89805bbbb065d047c2 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/test_python.sh' 1 jobs: ['7341051'] suites: ['1-ceph', '2-logrotate}', 'clusters/1-mds-1-client-coloc', 'conf/{client', 'distro/{rhel_8}', 'fs/libcephfs/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'tasks/libcephfs_python}'] Failure: error during scrub thrashing: reached maximum tries (31) after waiting for 900 seconds 3 jobs: ['7341046', '7341034', '7341053'] suites intersection: ['2-logrotate}'] suites union: ['1-ceph', '1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/yes', '3-snaps/yes', '4-flush/yes', '5-workunit/fs/norstats}}', '5-workunit/postgres}}', 'begin/{0-install', 'cephfs-mirror/three-per-cluster', 'clients/{mirror}', 'cluster/{1-node}', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs/mirror-ha/{begin/{0-install', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v1}', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/crc', 'n/3', 'n/5', 'objectstore-ec/bluestore-bitmap', 'objectstore/bluestore-bitmap', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/stock/{k-stock', 'overrides/{whitelist_health}', 'ranks/multi/{export-check', 'replication/always}', 'replication/default}', 'rhel_8}', 'session_timeout}', 'standby-replay', 'supported-random-distro$/{centos_8}', 'tasks/{0-subvolume/{no-subvolume}', 'tasks/{0-subvolume/{with-namespace-isolated}', 'workloads/cephfs-mirror-ha-workunit}', 'wsync/no}'] Failure: Test failure: test_snapshot_remove (tasks.cephfs.test_strays.TestStrays) 1 jobs: ['7341049'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'tasks/strays}'] Failure: Test failure: test_pool_perm (tasks.cephfs.test_pool_perm.TestPoolPerm) 1 jobs: ['7341043'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_8}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{ignorelist_health', 'tasks/pool-perm}'] Failure: Test failure: test_clean_stale_subvolume_snapshot_metadata (tasks.cephfs.test_volumes.TestSubvolumeSnapshots) 1 jobs: ['7341030'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_8}', 'fs/volumes/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'tasks/volumes/{overrides', 'test/snapshot}}'] Dead: b'2023-07-16T22:47:07.560 DEBUG:teuthology.exit:Finished running handlers' 1 jobs: ['7341028'] suites: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/no', '3-snaps/no', '4-flush/yes', '5-workunit/suites/blogbench}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'ms_mode/secure', 'n/3', 'objectstore-ec/bluestore-comp', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/stock/{k-stock', 'ranks/multi/{export-check', 'replication/default}', 'rhel_8}', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-no-extra-options}', 'wsync/no}'] Valgrind: mon (InvalidRead, Leak_PossiblyLost, Leak_StillReachable) 2 jobs: ['7341040', '7341050'] suites intersection: ['1-ceph', '2-logrotate}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{ubuntu/{latest', 'fs/verify/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon-debug', 'mount', 'mount/kclient/{k-testing', 'ms-die-on-skipped}', 'osd}', 'overrides/{ignorelist_health', 'overrides}}', 'session_timeout}', 'tasks/fsstress', 'validater/valgrind}'] suites union: ['1-ceph', '2-logrotate}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{ubuntu/{latest', 'fs/verify/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon-debug', 'mount', 'mount/kclient/{k-testing', 'ms-die-on-skipped}', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'osd}', 'overrides/{ignorelist_health', 'overrides}}', 'ranks/3', 'ranks/5', 'session_timeout}', 'tasks/fsstress', 'validater/valgrind}'] Failure: Test failure: test_client_release_bug (tasks.cephfs.test_client_limits.TestClientLimits) 1 jobs: ['7341035'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_20.04}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'tasks/client-limits}']