Found 99 jobs Missing teuthology log /home/teuthworker/archive/vshankar-2023-12-04_05:06:51-fs-wip-vshankar-testing-20231127.102654-distro-default-smithi/7476711/teuthology.log Missing teuthology log /home/teuthworker/archive/vshankar-2023-12-04_05:06:51-fs-wip-vshankar-testing-20231127.102654-distro-default-smithi/7476711/teuthology.log Missing teuthology log /home/teuthworker/archive/vshankar-2023-12-04_05:06:51-fs-wip-vshankar-testing-20231127.102654-distro-default-smithi/7476690/teuthology.log Misunderstood line: Missing teuthology log /home/teuthworker/archive/vshankar-2023-12-04_05:06:51-fs-wip-vshankar-testing-20231127.102654-distro-default-smithi/7476682/teuthology.log Misunderstood line: Misunderstood line: Missing teuthology log /home/teuthworker/archive/vshankar-2023-12-04_05:06:51-fs-wip-vshankar-testing-20231127.102654-distro-default-smithi/7476649/teuthology.log Found 30 distinct failure reasons Failure: Command failed on smithi164 with status 1: 'sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse ceph-volume librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd python3-cephfs cephfs-top cephfs-mirror bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib' 50 jobs suites intersection: ['2-logrotate}', 'conf/{client', 'mds', 'mon', 'osd}'] suites union: ['1-ceph', '1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/no', '2-scrub/yes', '2-workunit/cfuse_workunit_suites_fsstress}}', '2-workunit/cfuse_workunit_suites_pjd}}', '2-workunit/ffsb}}', '2-workunit/iozone}}', '2-workunit/suites/fsstress}}', '2-workunit/suites/pjd}}', '3-snaps/no', '3-snaps/yes', '4-flush/no', '4-flush/yes', '5-workunit/direct_io}}', '5-workunit/fs/misc}}', '5-workunit/fs/norstats}}', '5-workunit/fs/test_o_trunc}}', '5-workunit/kernel_untar_build}}', '5-workunit/postgres}}', '5-workunit/suites/blogbench}}', '5-workunit/suites/dbench}}', '5-workunit/suites/ffsb}}', '5-workunit/suites/fsstress}}', '5-workunit/suites/fsx}}', '5-workunit/suites/fsync-tester}}', '5-workunit/suites/iogen}}', '5-workunit/suites/pjd}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'clusters/1a3s-mds-1c-client', 'clusters/1a3s-mds-2c-client', 'clusters/1a3s-mds-4c-client', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{centos_latest}', 'distro/{ubuntu/{overrides', 'distro/{ubuntu_latest}', 'frag', 'fs/functional/{begin/{0-install', 'fs/mixed-clients/{begin/{0-install', 'fs/multifs/{begin/{0-install', 'fs/snaps/{begin/{0-install', 'fs/thrash/multifs/{begin/{0-install', 'fs/thrash/workloads/{begin/{0-install', 'fs/volumes/{begin/{0-install', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'ignorelist_wrongly_marked_down}', 'kclient-overrides/{distro/stock/{k-stock', 'mds', 'mon', 'mon-debug}', 'mount', 'mount/fuse', 'mount/kclient/{base/{mount-syntax/{v1}', 'mount/kclient/{base/{mount-syntax/{v2}', 'mount/kclient/{mount-syntax/{v1}', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}', 'ms-die-on-skipped}}', 'ms_mode/crc', 'ms_mode/legacy', 'ms_mode/secure', 'msgr-failures/none', 'msgr-failures/osd-mds-delay', 'multifs', 'n/3', 'n/5', 'no_client_pidfile}', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-ec-root', 'omap_limit/10', 'omap_limit/10000', 'osd-asserts', 'osd-asserts}', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{client-shutdown', 'overrides/{distro/stock/{k-stock', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'prefetch_dirfrags/no', 'prefetch_dirfrags/yes', 'prefetch_entire_dirfrags/no', 'prefetch_entire_dirfrags/yes', 'races', 'ranks/1', 'ranks/multi/{export-check', 'replication/always}', 'replication/default}', 'rhel_8}', 'session_timeout', 'session_timeout}', 'standby-replay', 'tasks/admin}', 'tasks/auto-repair}', 'tasks/client-limits}', 'tasks/damage}', 'tasks/forward-scrub}', 'tasks/kernel_cfuse_workunits_untarbuild_blogbench}', 'tasks/mds-flush}', 'tasks/mds_creation_retry}', 'tasks/metrics}', 'tasks/multifs-auth}', 'tasks/openfiletable}', 'tasks/pool-perm}', 'tasks/recovery-fs}', 'tasks/snap-schedule}', 'tasks/strays}', 'tasks/volumes/{overrides', 'tasks/workunit/dir-max-entries}', 'tasks/workunit/snaps}', 'tasks/{0-subvolume/{no-subvolume}', 'tasks/{0-subvolume/{with-namespace-isolated-and-quota}', 'tasks/{0-subvolume/{with-namespace-isolated}', 'tasks/{0-subvolume/{with-no-extra-options}', 'tasks/{0-subvolume/{with-quota}', 'tasks/{1-thrash/mds', 'tasks/{1-thrash/mon', 'tasks/{1-thrash/osd', 'test/basic}}', 'test/misc}}', 'thrashosds-health}', 'ubuntu_latest}}', 'wsync/no}', 'wsync/yes}'] Failure: Test failure: test_r_with_fsname_and_no_path_in_cap (tasks.cephfs.test_multifs_auth.TestMDSCaps) 1 jobs: ['7476632'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-2c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/multifs/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon-debug}', 'mount', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'objectstore-ec/bluestore-bitmap', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'tasks/multifs-auth}'] Failure: Test failure: test_export_pin_many (tasks.cephfs.test_exports.TestExportPin), test_export_pin_many (tasks.cephfs.test_exports.TestExportPin) 1 jobs: ['7476688'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{ignorelist_health', 'tasks/exports}'] Assertion: 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x188) [0x7f5eb116ef52] ceph version 18.0.0-7531-g47c1c547 (47c1c547372ed5bff7984647dc200bac0e019caf) reef (dev) 1: /lib64/libc.so.6(+0x54db0) [0x7f5eb0854db0] 2: /lib64/libc.so.6(+0xa154c) [0x7f5eb08a154c] 3: raise() 4: abort() 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x188) [0x7f5eb116ef52] 6: /usr/lib64/ceph/libceph-common.so.2(+0x16f0b6) [0x7f5eb116f0b6] 7: (Locker::xlock_start(SimpleLock*, boost::intrusive_ptr&)+0x939) [0x561c77098329] 8: (Locker::acquire_locks(boost::intrusive_ptr&, MutationImpl::LockOpVec&, CInode*, bool)+0x20f5) [0x561c7708d5b5] 9: (MDCache::path_traverse(boost::intrusive_ptr&, ContextFactory&, filepath const&, int, std::vector >*, CInode**)+0x79b) [0x561c7703c85b] 10: (Server::rdlock_path_xlock_dentry(boost::intrusive_ptr&, bool, bool, bool, bool)+0x371) [0x561c76f76521] 11: (Server::handle_client_unlink(boost::intrusive_ptr&)+0x90) [0x561c76f98070] 12: (Server::handle_peer_auth_pin_ack(boost::intrusive_ptr&, boost::intrusive_ptr const&)+0x83d) [0x561c76f72edd] 13: (Server::handle_peer_request_reply(boost::intrusive_ptr const&)+0x6bc) [0x561c76f737bc] 14: (Server::dispatch(boost::intrusive_ptr const&)+0x97) [0x561c76f5f467] 15: (MDSRank::_dispatch(boost::intrusive_ptr const&, bool)+0x4eb) [0x561c76ee348b] 16: (MDSRankDispatcher::ms_dispatch(boost::intrusive_ptr const&)+0x5c) [0x561c76ee397c] 17: (MDSDaemon::ms_dispatch2(boost::intrusive_ptr const&)+0x195) [0x561c76ecd4b5] 18: (DispatchQueue::entry()+0x53a) [0x7f5eb136040a] 19: /usr/lib64/ceph/libceph-common.so.2(+0x3f24a1) [0x7f5eb13f24a1] 20: /lib64/libc.so.6(+0x9f802) [0x7f5eb089f802] 21: /lib64/libc.so.6(+0x3f450) [0x7f5eb083f450] 1 jobs: ['7476657'] suites: ['1-ceph', '2-logrotate}', '2-workunit/suites/ffsb}}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{centos_latest}', 'frag', 'fs/thrash/workloads/{begin/{0-install', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'msgr-failures/osd-mds-delay', 'objectstore-ec/bluestore-comp', 'osd}', 'overrides/{client-shutdown', 'prefetch_dirfrags/no', 'prefetch_entire_dirfrags/no', 'races', 'ranks/5', 'session_timeout', 'tasks/{1-thrash/mon', 'thrashosds-health}'] Dead: None 4 jobs: ['7476711', '7476690', '7476682', '7476649'] Failure: Command failed on smithi018 with status 1: "sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:47c1c547372ed5bff7984647dc200bac0e019caf shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8f3ba672-9267-11ee-95a2-87774f69a715 -- bash -c 'ceph fs volume rename foo bar --yes-i-really-mean-it'" 1 jobs: ['7476646'] suites: ['1-rename', 'distro/single-container-host', 'fs/cephadm/renamevolume/{0-start', 'overrides/ignorelist_health}'] Assertion: 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x17b) [0x7ff4d2105ef0] ceph version 18.0.0-7531-g47c1c547 (47c1c547372ed5bff7984647dc200bac0e019caf) reef (dev) 1: /lib/x86_64-linux-gnu/libc.so.6(+0x42520) [0x7ff4d1a40520] 2: pthread_kill() 3: raise() 4: abort() 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x17b) [0x7ff4d2105ef0] 6: /usr/lib/ceph/libceph-common.so.2(+0x17004a) [0x7ff4d210604a] 7: (Locker::xlock_start(SimpleLock*, boost::intrusive_ptr&)+0x8d8) [0x5639b424d3e8] 8: (Locker::acquire_locks(boost::intrusive_ptr&, MutationImpl::LockOpVec&, CInode*, bool)+0x20b4) [0x5639b4242174] 9: (MDCache::path_traverse(boost::intrusive_ptr&, ContextFactory&, filepath const&, int, std::vector >*, CInode**)+0x8d3) [0x5639b41f3033] 10: (Server::rdlock_path_xlock_dentry(boost::intrusive_ptr&, bool, bool, bool, bool)+0x21c) [0x5639b412ddec] 11: (Server::handle_client_openc(boost::intrusive_ptr&)+0xd8) [0x5639b4134be8] 12: (Server::handle_peer_auth_pin_ack(boost::intrusive_ptr&, boost::intrusive_ptr const&)+0x801) [0x5639b412aa91] 13: (Server::handle_peer_request_reply(boost::intrusive_ptr const&)+0x6ac) [0x5639b412b34c] 14: (Server::dispatch(boost::intrusive_ptr const&)+0x97) [0x5639b4117837] 15: (MDSRank::_dispatch(boost::intrusive_ptr const&, bool)+0x4cb) [0x5639b409b5fb] 16: (MDSRankDispatcher::ms_dispatch(boost::intrusive_ptr const&)+0x5c) [0x5639b409bacc] 17: (MDSDaemon::ms_dispatch2(boost::intrusive_ptr const&)+0x195) [0x5639b40868e5] 18: (DispatchQueue::entry()+0x5ea) [0x7ff4d22f211a] 19: /usr/lib/ceph/libceph-common.so.2(+0x3ee991) [0x7ff4d2384991] 20: /lib/x86_64-linux-gnu/libc.so.6(+0x94b43) [0x7ff4d1a92b43] 21: /lib/x86_64-linux-gnu/libc.so.6(+0x126a00) [0x7ff4d1b24a00] 1 jobs: ['7476722'] suites: ['1-ceph', '2-logrotate}', '2-workunit/suites/ffsb}}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'frag', 'fs/thrash/workloads/{begin/{0-install', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'msgr-failures/osd-mds-delay', 'objectstore-ec/bluestore-comp', 'osd}', 'overrides/{client-shutdown', 'prefetch_dirfrags/no', 'prefetch_entire_dirfrags/no', 'races', 'ranks/5', 'session_timeout', 'tasks/{1-thrash/osd', 'thrashosds-health}'] Assertion: 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x17b) [0x7f7f72012ef0] ceph version 18.0.0-7531-g47c1c547 (47c1c547372ed5bff7984647dc200bac0e019caf) reef (dev) 1: /lib/x86_64-linux-gnu/libc.so.6(+0x42520) [0x7f7f7194d520] 2: pthread_kill() 3: raise() 4: abort() 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x17b) [0x7f7f72012ef0] 6: /usr/lib/ceph/libceph-common.so.2(+0x17004a) [0x7f7f7201304a] 7: (Locker::xlock_start(SimpleLock*, boost::intrusive_ptr&)+0x8d8) [0x55596777a3e8] 8: (Locker::acquire_locks(boost::intrusive_ptr&, MutationImpl::LockOpVec&, CInode*, bool)+0x20b4) [0x55596776f174] 9: (MDCache::path_traverse(boost::intrusive_ptr&, ContextFactory&, filepath const&, int, std::vector >*, CInode**)+0x8d3) [0x555967720033] 10: (Server::rdlock_path_xlock_dentry(boost::intrusive_ptr&, bool, bool, bool, bool)+0x21c) [0x55596765adec] 11: (Server::handle_client_openc(boost::intrusive_ptr&)+0xd8) [0x555967661be8] 12: (Server::handle_peer_auth_pin_ack(boost::intrusive_ptr&, boost::intrusive_ptr const&)+0x801) [0x555967657a91] 13: (Server::handle_peer_request_reply(boost::intrusive_ptr const&)+0x6ac) [0x55596765834c] 14: (Server::dispatch(boost::intrusive_ptr const&)+0x97) [0x555967644837] 15: (MDSRank::_dispatch(boost::intrusive_ptr const&, bool)+0x4cb) [0x5559675c85fb] 16: (MDSRankDispatcher::ms_dispatch(boost::intrusive_ptr const&)+0x5c) [0x5559675c8acc] 17: (MDSDaemon::ms_dispatch2(boost::intrusive_ptr const&)+0x195) [0x5559675b38e5] 18: (DispatchQueue::entry()+0x5ea) [0x7f7f721ff11a] 19: /usr/lib/ceph/libceph-common.so.2(+0x3ee991) [0x7f7f72291991] 20: /lib/x86_64-linux-gnu/libc.so.6(+0x94b43) [0x7f7f7199fb43] 21: /lib/x86_64-linux-gnu/libc.so.6(+0x126a00) [0x7f7f71a31a00] 1 jobs: ['7476720'] suites: ['1-ceph', '2-logrotate}', '2-workunit/fs/snaps}}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'frag', 'fs/thrash/workloads/{begin/{0-install', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'msgr-failures/none', 'objectstore-ec/bluestore-comp-ec-root', 'osd}', 'overrides/{client-shutdown', 'overrides/{distro/testing/k-testing', 'prefetch_dirfrags/no', 'prefetch_entire_dirfrags/yes', 'races', 'ranks/3', 'session_timeout', 'tasks/{1-thrash/mon', 'thrashosds-health}'] Assertion: 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x121) [0x4bd7e96] ceph version 18.0.0-7531-g47c1c547 (47c1c547372ed5bff7984647dc200bac0e019caf) reef (dev) 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x121) [0x4bd7e96] 2: /usr/lib/ceph/libceph-common.so.2(+0x17004a) [0x4bd804a] 3: (Locker::xlock_start(SimpleLock*, boost::intrusive_ptr&)+0x8d8) [0x4003e8] 4: (Locker::acquire_locks(boost::intrusive_ptr&, MutationImpl::LockOpVec&, CInode*, bool)+0x20b4) [0x3f5174] 5: (MDCache::path_traverse(boost::intrusive_ptr&, ContextFactory&, filepath const&, int, std::vector >*, CInode**)+0x8d3) [0x3a6033] 6: (Server::rdlock_path_xlock_dentry(boost::intrusive_ptr&, bool, bool, bool, bool)+0x21c) [0x2e0dec] 7: (Server::handle_client_openc(boost::intrusive_ptr&)+0xd8) [0x2e7be8] 8: (Server::handle_peer_auth_pin_ack(boost::intrusive_ptr&, boost::intrusive_ptr const&)+0x801) [0x2dda91] 9: (Server::handle_peer_request_reply(boost::intrusive_ptr const&)+0x6ac) [0x2de34c] 10: (Server::dispatch(boost::intrusive_ptr const&)+0x97) [0x2ca837] 11: (MDSRank::_dispatch(boost::intrusive_ptr const&, bool)+0x4cb) [0x24e5fb] 12: (MDSRankDispatcher::ms_dispatch(boost::intrusive_ptr const&)+0x5c) [0x24eacc] 13: (MDSDaemon::ms_dispatch2(boost::intrusive_ptr const&)+0x195) [0x2398e5] 14: (DispatchQueue::entry()+0x5ea) [0x4dc411a] 15: /usr/lib/ceph/libceph-common.so.2(+0x3ee991) [0x4e56991] 16: /lib/x86_64-linux-gnu/libc.so.6(+0x94b43) [0x57a5b43] 2 jobs: ['7476672', '7476635'] suites intersection: ['1-ceph', '2-logrotate}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{ubuntu/{overrides', 'fs/verify/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon-debug', 'mount', 'mount/kclient/{k-testing', 'ms-die-on-skipped}', 'osd}', 'overrides/{ignorelist_health', 'session_timeout}', 'tasks/fsstress', 'ubuntu_latest}}', 'validater/valgrind}'] suites union: ['1-ceph', '2-logrotate}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{ubuntu/{overrides', 'fs/verify/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon-debug', 'mount', 'mount/kclient/{k-testing', 'ms-die-on-skipped}', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'osd}', 'overrides/{ignorelist_health', 'ranks/3', 'ranks/5', 'session_timeout}', 'tasks/fsstress', 'ubuntu_latest}}', 'validater/valgrind}'] Failure: Test failure: test_generic (tasks.cephfs.tests_from_xfstests_dev.TestXFSTestsDev) 1 jobs: ['7476656'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'tasks/xfstests-dev}'] Failure: Test failure: test_replay_beacon_estimated_time (tasks.cephfs.test_failover.TestFailoverBeaconHealth) 1 jobs: ['7476679'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-2c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/multifs/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon-debug}', 'mount', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'objectstore-ec/bluestore-ec-root', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'tasks/failover}'] Assertion: 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x188) [0x7fa7e056ef52] ceph version 18.0.0-7531-g47c1c547 (47c1c547372ed5bff7984647dc200bac0e019caf) reef (dev) 1: /lib64/libc.so.6(+0x54db0) [0x7fa7dfc54db0] 2: /lib64/libc.so.6(+0xa154c) [0x7fa7dfca154c] 3: raise() 4: abort() 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x188) [0x7fa7e056ef52] 6: /usr/lib64/ceph/libceph-common.so.2(+0x16f0b6) [0x7fa7e056f0b6] 7: (Locker::xlock_start(SimpleLock*, boost::intrusive_ptr&)+0x939) [0x55a92b5f7329] 8: (Locker::acquire_locks(boost::intrusive_ptr&, MutationImpl::LockOpVec&, CInode*, bool)+0x20f5) [0x55a92b5ec5b5] 9: (MDCache::path_traverse(boost::intrusive_ptr&, ContextFactory&, filepath const&, int, std::vector >*, CInode**)+0x79b) [0x55a92b59b85b] 10: (Server::rdlock_path_xlock_dentry(boost::intrusive_ptr&, bool, bool, bool, bool)+0x371) [0x55a92b4d5521] 11: (Server::handle_client_openc(boost::intrusive_ptr&)+0xd8) [0x55a92b4dc508] 12: (Server::handle_peer_auth_pin_ack(boost::intrusive_ptr&, boost::intrusive_ptr const&)+0x83d) [0x55a92b4d1edd] 13: (Server::handle_peer_request_reply(boost::intrusive_ptr const&)+0x6bc) [0x55a92b4d27bc] 14: (Server::dispatch(boost::intrusive_ptr const&)+0x97) [0x55a92b4be467] 15: (MDSRank::_dispatch(boost::intrusive_ptr const&, bool)+0x4eb) [0x55a92b44248b] 16: (MDSRankDispatcher::ms_dispatch(boost::intrusive_ptr const&)+0x5c) [0x55a92b44297c] 17: (MDSDaemon::ms_dispatch2(boost::intrusive_ptr const&)+0x195) [0x55a92b42c4b5] 18: (DispatchQueue::entry()+0x53a) [0x7fa7e076040a] 19: /usr/lib64/ceph/libceph-common.so.2(+0x3f24a1) [0x7fa7e07f24a1] 20: /lib64/libc.so.6(+0x9f802) [0x7fa7dfc9f802] 21: /lib64/libc.so.6(+0x3f450) [0x7fa7dfc3f450] 1 jobs: ['7476641'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_latest}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'tasks/sessionmap}'] Assertion: 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x17b) [0x7fb3b09fdef0] ceph version 18.0.0-7531-g47c1c547 (47c1c547372ed5bff7984647dc200bac0e019caf) reef (dev) 1: /lib/x86_64-linux-gnu/libc.so.6(+0x42520) [0x7fb3b0338520] 2: pthread_kill() 3: raise() 4: abort() 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x17b) [0x7fb3b09fdef0] 6: /usr/lib/ceph/libceph-common.so.2(+0x17004a) [0x7fb3b09fe04a] 7: (Locker::xlock_start(SimpleLock*, boost::intrusive_ptr&)+0x8d8) [0x558329c363e8] 8: (Locker::acquire_locks(boost::intrusive_ptr&, MutationImpl::LockOpVec&, CInode*, bool)+0x20b4) [0x558329c2b174] 9: (MDCache::path_traverse(boost::intrusive_ptr&, ContextFactory&, filepath const&, int, std::vector >*, CInode**)+0x8d3) [0x558329bdc033] 10: (Server::rdlock_path_xlock_dentry(boost::intrusive_ptr&, bool, bool, bool, bool)+0x21c) [0x558329b16dec] 11: (Server::handle_client_openc(boost::intrusive_ptr&)+0xd8) [0x558329b1dbe8] 12: (Server::handle_peer_auth_pin_ack(boost::intrusive_ptr&, boost::intrusive_ptr const&)+0x801) [0x558329b13a91] 13: (Server::handle_peer_request_reply(boost::intrusive_ptr const&)+0x6ac) [0x558329b1434c] 14: (Server::dispatch(boost::intrusive_ptr const&)+0x97) [0x558329b00837] 15: (MDSRank::_dispatch(boost::intrusive_ptr const&, bool)+0x4cb) [0x558329a845fb] 16: (MDSRankDispatcher::ms_dispatch(boost::intrusive_ptr const&)+0x5c) [0x558329a84acc] 17: (MDSDaemon::ms_dispatch2(boost::intrusive_ptr const&)+0x195) [0x558329a6f8e5] 18: (DispatchQueue::entry()+0x5ea) [0x7fb3b0bea11a] 19: /usr/lib/ceph/libceph-common.so.2(+0x3ee991) [0x7fb3b0c7c991] 20: /lib/x86_64-linux-gnu/libc.so.6(+0x94b43) [0x7fb3b038ab43] 21: /lib/x86_64-linux-gnu/libc.so.6(+0x126a00) [0x7fb3b041ca00] 1 jobs: ['7476702'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'tasks/journal-repair}'] Failure: Test failure: test_subvolume_snapshot_clone_with_no_wait_enabled (tasks.cephfs.test_volumes.TestSubvolumeSnapshotClones) 1 jobs: ['7476644'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/volumes/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'tasks/volumes/{overrides', 'test/clone}}'] Assertion: 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x17b) [0x7f9a65ebaef0] ceph version 18.0.0-7531-g47c1c547 (47c1c547372ed5bff7984647dc200bac0e019caf) reef (dev) 1: /lib/x86_64-linux-gnu/libc.so.6(+0x42520) [0x7f9a657f5520] 2: pthread_kill() 3: raise() 4: abort() 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x17b) [0x7f9a65ebaef0] 6: /usr/lib/ceph/libceph-common.so.2(+0x17004a) [0x7f9a65ebb04a] 7: (MDSTableServer::handle_prepare(boost::intrusive_ptr const&)+0x341) [0x5561a4a45d11] 8: (MDSRank::handle_message(boost::intrusive_ptr const&)+0xe9) [0x5561a47de579] 9: (MDSRank::_dispatch(boost::intrusive_ptr const&, bool)+0x4cb) [0x5561a47e05fb] 10: (MDSRankDispatcher::ms_dispatch(boost::intrusive_ptr const&)+0x5c) [0x5561a47e0acc] 11: (MDSDaemon::ms_dispatch2(boost::intrusive_ptr const&)+0x195) [0x5561a47cb8e5] 12: (DispatchQueue::entry()+0x5ea) [0x7f9a660a711a] 13: /usr/lib/ceph/libceph-common.so.2(+0x3ee991) [0x7f9a66139991] 14: /lib/x86_64-linux-gnu/libc.so.6(+0x94b43) [0x7f9a65847b43] 15: /lib/x86_64-linux-gnu/libc.so.6(+0x126a00) [0x7f9a658d9a00] 1 jobs: ['7476648'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{ignorelist_health', 'tasks/snapshots}'] Assertion: 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x188) [0x7fe5a616ef52] ceph version 18.0.0-7531-g47c1c547 (47c1c547372ed5bff7984647dc200bac0e019caf) reef (dev) 1: /lib64/libc.so.6(+0x54db0) [0x7fe5a5854db0] 2: /lib64/libc.so.6(+0xa154c) [0x7fe5a58a154c] 3: raise() 4: abort() 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x188) [0x7fe5a616ef52] 6: /usr/lib64/ceph/libceph-common.so.2(+0x16f0b6) [0x7fe5a616f0b6] 7: (Locker::xlock_start(SimpleLock*, boost::intrusive_ptr&)+0x939) [0x56302cfc1329] 8: (Locker::acquire_locks(boost::intrusive_ptr&, MutationImpl::LockOpVec&, CInode*, bool)+0x20f5) [0x56302cfb65b5] 9: (MDCache::path_traverse(boost::intrusive_ptr&, ContextFactory&, filepath const&, int, std::vector >*, CInode**)+0x79b) [0x56302cf6585b] 10: (Server::rdlock_path_xlock_dentry(boost::intrusive_ptr&, bool, bool, bool, bool)+0x371) [0x56302ce9f521] 11: (Server::handle_client_unlink(boost::intrusive_ptr&)+0x90) [0x56302cec1070] 12: (Server::handle_peer_auth_pin_ack(boost::intrusive_ptr&, boost::intrusive_ptr const&)+0x83d) [0x56302ce9bedd] 13: (Server::handle_peer_request_reply(boost::intrusive_ptr const&)+0x6bc) [0x56302ce9c7bc] 14: (Server::dispatch(boost::intrusive_ptr const&)+0x97) [0x56302ce88467] 15: (MDSRank::_dispatch(boost::intrusive_ptr const&, bool)+0x4eb) [0x56302ce0c48b] 16: (MDSRankDispatcher::ms_dispatch(boost::intrusive_ptr const&)+0x5c) [0x56302ce0c97c] 17: (MDSDaemon::ms_dispatch2(boost::intrusive_ptr const&)+0x195) [0x56302cdf64b5] 18: (DispatchQueue::entry()+0x53a) [0x7fe5a636040a] 19: /usr/lib64/ceph/libceph-common.so.2(+0x3f24a1) [0x7fe5a63f24a1] 20: /lib64/libc.so.6(+0x9f802) [0x7fe5a589f802] 21: /lib64/libc.so.6(+0x3f450) [0x7fe5a583f450] 1 jobs: ['7476628'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{centos_latest}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'tasks/metrics}'] Failure: Test failure: test_rebuild_inotable (tasks.cephfs.test_data_scan.TestDataScan), test_rebuild_inotable (tasks.cephfs.test_data_scan.TestDataScan) 1 jobs: ['7476685'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{distro/testing/k-testing', 'overrides/{ignorelist_health', 'tasks/data-scan}'] Valgrind: mon (Leak_StillReachable) 1 jobs: ['7476713'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{ubuntu/{overrides', 'fs/verify/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon-debug', 'mount', 'mount/kclient/{k-testing', 'ms-die-on-skipped}', 'objectstore-ec/bluestore-comp', 'osd}', 'overrides/{ignorelist_health', 'ranks/1', 'session_timeout}', 'tasks/fsstress', 'ubuntu_latest}}', 'validater/valgrind}'] Assertion: 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x188) [0x7ff993f6ef52] ceph version 18.0.0-7531-g47c1c547 (47c1c547372ed5bff7984647dc200bac0e019caf) reef (dev) 1: /lib64/libc.so.6(+0x54db0) [0x7ff993654db0] 2: /lib64/libc.so.6(+0xa154c) [0x7ff9936a154c] 3: raise() 4: abort() 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x188) [0x7ff993f6ef52] 6: /usr/lib64/ceph/libceph-common.so.2(+0x16f0b6) [0x7ff993f6f0b6] 7: (Locker::xlock_start(SimpleLock*, boost::intrusive_ptr&)+0x939) [0x561c7d608329] 8: (Locker::acquire_locks(boost::intrusive_ptr&, MutationImpl::LockOpVec&, CInode*, bool)+0x20f5) [0x561c7d5fd5b5] 9: (MDCache::path_traverse(boost::intrusive_ptr&, ContextFactory&, filepath const&, int, std::vector >*, CInode**)+0x79b) [0x561c7d5ac85b] 10: (Server::rdlock_path_xlock_dentry(boost::intrusive_ptr&, bool, bool, bool, bool)+0x371) [0x561c7d4e6521] 11: (Server::handle_client_openc(boost::intrusive_ptr&)+0xd8) [0x561c7d4ed508] 12: (Server::handle_peer_auth_pin_ack(boost::intrusive_ptr&, boost::intrusive_ptr const&)+0x83d) [0x561c7d4e2edd] 13: (Server::handle_peer_request_reply(boost::intrusive_ptr const&)+0x6bc) [0x561c7d4e37bc] 14: (Server::dispatch(boost::intrusive_ptr const&)+0x97) [0x561c7d4cf467] 15: (MDSRank::_dispatch(boost::intrusive_ptr const&, bool)+0x4eb) [0x561c7d45348b] 16: (MDSRankDispatcher::ms_dispatch(boost::intrusive_ptr const&)+0x5c) [0x561c7d45397c] 17: (MDSDaemon::ms_dispatch2(boost::intrusive_ptr const&)+0x195) [0x561c7d43d4b5] 18: (DispatchQueue::entry()+0x53a) [0x7ff99416040a] 19: /usr/lib64/ceph/libceph-common.so.2(+0x3f24a1) [0x7ff9941f24a1] 20: /lib64/libc.so.6(+0x9f802) [0x7ff99369f802] 21: /lib64/libc.so.6(+0x3f450) [0x7ff99363f450] 1 jobs: ['7476630'] suites: ['1-ceph', '2-logrotate}', '2-workunit/suites/ffsb}}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{centos_latest}', 'frag', 'fs/thrash/workloads/{begin/{0-install', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'msgr-failures/osd-mds-delay', 'objectstore-ec/bluestore-bitmap', 'osd}', 'overrides/{client-shutdown', 'prefetch_dirfrags/no', 'prefetch_entire_dirfrags/yes', 'races', 'ranks/5', 'session_timeout', 'tasks/{1-thrash/osd', 'thrashosds-health}'] Dead: b'2023-12-04T17:21:46.998 DEBUG:teuthology.exit:Finished running handlers' 2 jobs: ['7476647', '7476659'] suites intersection: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-2c-client', 'conf/{client', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd}', 'overrides/{ignorelist_health'] suites union: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-2c-client', 'conf/{client', 'distro/{centos_latest}', 'distro/{ubuntu/{overrides', 'fs/mixed-clients/{begin/{0-install', 'fs/multifs/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'kclient-overrides/{distro/testing/k-testing', 'mds', 'mon', 'mon-debug}', 'mount/fuse', 'ms-die-on-skipped}', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'osd-asserts}', 'osd}', 'overrides/{ignorelist_health', 'tasks/failover}', 'tasks/kernel_cfuse_workunits_dbench_iozone}', 'ubuntu_latest}}'] Assertion: 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x17b) [0x7f747756fef0] ceph version 18.0.0-7531-g47c1c547 (47c1c547372ed5bff7984647dc200bac0e019caf) reef (dev) 1: /lib/x86_64-linux-gnu/libc.so.6(+0x42520) [0x7f7476c42520] 2: pthread_kill() 3: raise() 4: abort() 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x17b) [0x7f747756fef0] 6: /usr/lib/ceph/libceph-common.so.2(+0x17004a) [0x7f747757004a] 7: (Locker::xlock_start(SimpleLock*, boost::intrusive_ptr&)+0x8d8) [0x55a8f00eb3e8] 8: (Locker::acquire_locks(boost::intrusive_ptr&, MutationImpl::LockOpVec&, CInode*, bool)+0x20b4) [0x55a8f00e0174] 9: (MDCache::path_traverse(boost::intrusive_ptr&, ContextFactory&, filepath const&, int, std::vector >*, CInode**)+0x8d3) [0x55a8f0091033] 10: (Server::rdlock_path_xlock_dentry(boost::intrusive_ptr&, bool, bool, bool, bool)+0x21c) [0x55a8effcbdec] 11: (Server::handle_client_openc(boost::intrusive_ptr&)+0xd8) [0x55a8effd2be8] 12: (Server::handle_peer_auth_pin_ack(boost::intrusive_ptr&, boost::intrusive_ptr const&)+0x801) [0x55a8effc8a91] 13: (Server::handle_peer_request_reply(boost::intrusive_ptr const&)+0x6ac) [0x55a8effc934c] 14: (Server::dispatch(boost::intrusive_ptr const&)+0x97) [0x55a8effb5837] 15: (MDSRank::_dispatch(boost::intrusive_ptr const&, bool)+0x4cb) [0x55a8eff395fb] 16: (MDSRankDispatcher::ms_dispatch(boost::intrusive_ptr const&)+0x5c) [0x55a8eff39acc] 17: (MDSDaemon::ms_dispatch2(boost::intrusive_ptr const&)+0x195) [0x55a8eff248e5] 18: (DispatchQueue::entry()+0x5ea) [0x7f747775c11a] 19: /usr/lib/ceph/libceph-common.so.2(+0x3ee991) [0x7f74777ee991] 20: /lib/x86_64-linux-gnu/libc.so.6(+0x94b43) [0x7f7476c94b43] 21: /lib/x86_64-linux-gnu/libc.so.6(+0x126a00) [0x7f7476d26a00] 1 jobs: ['7476653'] suites: ['1-ceph', '2-logrotate}', '2-workunit/fs/snaps}}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'frag', 'fs/thrash/workloads/{begin/{0-install', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'msgr-failures/none', 'objectstore-ec/bluestore-comp-ec-root', 'osd}', 'overrides/{client-shutdown', 'overrides/{distro/testing/k-testing', 'prefetch_dirfrags/no', 'prefetch_entire_dirfrags/yes', 'races', 'ranks/3', 'session_timeout', 'tasks/{1-thrash/mds', 'thrashosds-health}'] Failure: Test failure: test_basic (tasks.cephfs.test_dump_tree.TestDumpTree) 1 jobs: ['7476669'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/functional/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'tasks/asok_dump_tree}'] Failure: [Errno None] Unable to connect to port 22 on 172.21.15.88 1 jobs: ['7476683'] suites: ['1-cephadm', '1-check-counter', '2-logrotate}', '2-scrub/yes', '3-snaps/yes', '4-flush/yes', '5-workunit/suites/iozone}}', 'begin/{0-install', 'clusters/1a11s-mds-1c-client-3node', 'conf/{client', 'frag', 'fs/workload/{0-rhel_8', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{base/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'ms_mode/secure', 'objectstore-ec/bluestore-ec-root', 'omap_limit/10', 'osd-asserts', 'osd}', 'overrides/{cephsqlite-timeout', 'overrides/{distro/testing/k-testing', 'ranks/1', 'session_timeout}', 'standby-replay', 'tasks/{0-subvolume/{with-quota}', 'wsync/yes}'] Assertion: 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x188) [0x7f24b576ef52] ceph version 18.0.0-7531-g47c1c547 (47c1c547372ed5bff7984647dc200bac0e019caf) reef (dev) 1: /lib64/libc.so.6(+0x54db0) [0x7f24b4e54db0] 2: /lib64/libc.so.6(+0xa154c) [0x7f24b4ea154c] 3: raise() 4: abort() 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x188) [0x7f24b576ef52] 6: /usr/lib64/ceph/libceph-common.so.2(+0x16f0b6) [0x7f24b576f0b6] 7: (Locker::xlock_start(SimpleLock*, boost::intrusive_ptr&)+0x939) [0x55dd97e17329] 8: (Locker::acquire_locks(boost::intrusive_ptr&, MutationImpl::LockOpVec&, CInode*, bool)+0x20f5) [0x55dd97e0c5b5] 9: (MDCache::path_traverse(boost::intrusive_ptr&, ContextFactory&, filepath const&, int, std::vector >*, CInode**)+0x79b) [0x55dd97dbb85b] 10: (Server::rdlock_path_xlock_dentry(boost::intrusive_ptr&, bool, bool, bool, bool)+0x371) [0x55dd97cf5521] 11: (Server::handle_client_openc(boost::intrusive_ptr&)+0xd8) [0x55dd97cfc508] 12: (Server::handle_peer_auth_pin_ack(boost::intrusive_ptr&, boost::intrusive_ptr const&)+0x83d) [0x55dd97cf1edd] 13: (Server::handle_peer_request_reply(boost::intrusive_ptr const&)+0x6bc) [0x55dd97cf27bc] 14: (Server::dispatch(boost::intrusive_ptr const&)+0x97) [0x55dd97cde467] 15: (MDSRank::_dispatch(boost::intrusive_ptr const&, bool)+0x4eb) [0x55dd97c6248b] 16: (MDSRankDispatcher::ms_dispatch(boost::intrusive_ptr const&)+0x5c) [0x55dd97c6297c] 17: (MDSDaemon::ms_dispatch2(boost::intrusive_ptr const&)+0x195) [0x55dd97c4c4b5] 18: (DispatchQueue::entry()+0x53a) [0x7f24b596040a] 19: /usr/lib64/ceph/libceph-common.so.2(+0x3f24a1) [0x7f24b59f24a1] 20: /lib64/libc.so.6(+0x9f802) [0x7f24b4e9f802] 21: /lib64/libc.so.6(+0x3f450) [0x7f24b4e3f450] 1 jobs: ['7476681'] suites: ['1-ceph', '2-logrotate}', '2-workunit/fs/snaps}}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{centos_latest}', 'frag', 'fs/thrash/workloads/{begin/{0-install', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v1}', 'ms-die-on-skipped}}', 'msgr-failures/none', 'objectstore-ec/bluestore-ec-root', 'osd}', 'overrides/{client-shutdown', 'overrides/{distro/testing/k-testing', 'prefetch_dirfrags/yes', 'prefetch_entire_dirfrags/yes', 'races', 'ranks/3', 'session_timeout', 'tasks/{1-thrash/osd', 'thrashosds-health}'] Failure: Test failure: test_subvolumegroup_pin_distributed (tasks.cephfs.test_volumes.TestSubvolumeGroups) 1 jobs: ['7476700'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/volumes/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'tasks/volumes/{overrides', 'test/basic}}'] Valgrind: cephfs-mirror-client (Leak_StillReachable) 1 jobs: ['7476699'] suites: ['1-ceph', '2-logrotate}', 'centos_latest', 'clients/mirror', 'cluster/1-node', 'debug', 'fs/valgrind/{begin/{0-install', 'mirror/{cephfs-mirror/one-per-cluster', 'mount/fuse', 'overrides/ignorelist_health', 'tasks/mirror}}'] Failure: SSH connection to smithi088 was lost: 'sudo apt-get update' 1 jobs: ['7476697'] suites: ['1-ceph', '2-logrotate}', 'clusters/1a3s-mds-2c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/multifs/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mon-debug}', 'mount/fuse', 'objectstore-ec/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'tasks/failover}'] Assertion: 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x17b) [0x7fc894deeef0] ceph version 18.0.0-7531-g47c1c547 (47c1c547372ed5bff7984647dc200bac0e019caf) reef (dev) 1: /lib/x86_64-linux-gnu/libc.so.6(+0x42520) [0x7fc894729520] 2: pthread_kill() 3: raise() 4: abort() 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x17b) [0x7fc894deeef0] 6: /usr/lib/ceph/libceph-common.so.2(+0x17004a) [0x7fc894def04a] 7: (Locker::xlock_start(SimpleLock*, boost::intrusive_ptr&)+0x8d8) [0x5615e73433e8] 8: (Locker::acquire_locks(boost::intrusive_ptr&, MutationImpl::LockOpVec&, CInode*, bool)+0x20b4) [0x5615e7338174] 9: (MDCache::path_traverse(boost::intrusive_ptr&, ContextFactory&, filepath const&, int, std::vector >*, CInode**)+0x8d3) [0x5615e72e9033] 10: (Server::rdlock_path_xlock_dentry(boost::intrusive_ptr&, bool, bool, bool, bool)+0x21c) [0x5615e7223dec] 11: (Server::handle_client_openc(boost::intrusive_ptr&)+0xd8) [0x5615e722abe8] 12: (Server::handle_peer_auth_pin_ack(boost::intrusive_ptr&, boost::intrusive_ptr const&)+0x801) [0x5615e7220a91] 13: (Server::handle_peer_request_reply(boost::intrusive_ptr const&)+0x6ac) [0x5615e722134c] 14: (Server::dispatch(boost::intrusive_ptr const&)+0x97) [0x5615e720d837] 15: (MDSRank::_dispatch(boost::intrusive_ptr const&, bool)+0x4cb) [0x5615e71915fb] 16: (MDSRankDispatcher::ms_dispatch(boost::intrusive_ptr const&)+0x5c) [0x5615e7191acc] 17: (MDSDaemon::ms_dispatch2(boost::intrusive_ptr const&)+0x195) [0x5615e717c8e5] 18: (DispatchQueue::entry()+0x5ea) [0x7fc894fdb11a] 19: /usr/lib/ceph/libceph-common.so.2(+0x3ee991) [0x7fc89506d991] 20: /lib/x86_64-linux-gnu/libc.so.6(+0x94b43) [0x7fc89477bb43] 21: /lib/x86_64-linux-gnu/libc.so.6(+0x126a00) [0x7fc89480da00] 1 jobs: ['7476627'] suites: ['1-ceph', '2-logrotate}', '2-workunit/fs/snaps}}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'frag', 'fs/thrash/workloads/{begin/{0-install', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount', 'mount/kclient/{mount-syntax/{v2}', 'ms-die-on-skipped}}', 'msgr-failures/none', 'objectstore-ec/bluestore-ec-root', 'osd}', 'overrides/{client-shutdown', 'overrides/{distro/testing/k-testing', 'prefetch_dirfrags/yes', 'prefetch_entire_dirfrags/no', 'races', 'ranks/3', 'session_timeout', 'tasks/{1-thrash/mon', 'thrashosds-health}'] Failure: Command failed (workunit test client/test.sh) on smithi114 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=47c1c547372ed5bff7984647dc200bac0e019caf TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/client/test.sh' 1 jobs: ['7476695'] suites: ['1-ceph', '2-logrotate}', 'clusters/1-mds-1-client-coloc', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/libcephfs/{begin/{0-install', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'no_client_pidfile}', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{ignorelist_health', 'tasks/client}'] Assertion: 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x17b) [0x7ff3c558fef0] ceph version 18.0.0-7531-g47c1c547 (47c1c547372ed5bff7984647dc200bac0e019caf) reef (dev) 1: /lib/x86_64-linux-gnu/libc.so.6(+0x42520) [0x7ff3c4eca520] 2: pthread_kill() 3: raise() 4: abort() 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x17b) [0x7ff3c558fef0] 6: /usr/lib/ceph/libceph-common.so.2(+0x17004a) [0x7ff3c559004a] 7: (Locker::xlock_start(SimpleLock*, boost::intrusive_ptr&)+0x8d8) [0x564b14b293e8] 8: (Locker::acquire_locks(boost::intrusive_ptr&, MutationImpl::LockOpVec&, CInode*, bool)+0x20b4) [0x564b14b1e174] 9: (MDCache::path_traverse(boost::intrusive_ptr&, ContextFactory&, filepath const&, int, std::vector >*, CInode**)+0x8d3) [0x564b14acf033] 10: (Server::rdlock_path_xlock_dentry(boost::intrusive_ptr&, bool, bool, bool, bool)+0x21c) [0x564b14a09dec] 11: (Server::handle_client_openc(boost::intrusive_ptr&)+0xd8) [0x564b14a10be8] 12: (Server::handle_peer_auth_pin_ack(boost::intrusive_ptr&, boost::intrusive_ptr const&)+0x801) [0x564b14a06a91] 13: (Server::handle_peer_request_reply(boost::intrusive_ptr const&)+0x6ac) [0x564b14a0734c] 14: (Server::dispatch(boost::intrusive_ptr const&)+0x97) [0x564b149f3837] 15: (MDSRank::_dispatch(boost::intrusive_ptr const&, bool)+0x4cb) [0x564b149775fb] 16: (MDSRankDispatcher::ms_dispatch(boost::intrusive_ptr const&)+0x5c) [0x564b14977acc] 17: (MDSDaemon::ms_dispatch2(boost::intrusive_ptr const&)+0x195) [0x564b149628e5] 18: (DispatchQueue::entry()+0x5ea) [0x7ff3c577c11a] 19: /usr/lib/ceph/libceph-common.so.2(+0x3ee991) [0x7ff3c580e991] 20: /lib/x86_64-linux-gnu/libc.so.6(+0x94b43) [0x7ff3c4f1cb43] 21: /lib/x86_64-linux-gnu/libc.so.6(+0x126a00) [0x7ff3c4faea00] 1 jobs: ['7476686'] suites: ['1-ceph', '2-logrotate}', '2-workunit/suites/ffsb}}', 'clusters/1a5s-mds-1c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'frag', 'fs/thrash/workloads/{begin/{0-install', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'mount/fuse', 'msgr-failures/osd-mds-delay', 'objectstore-ec/bluestore-bitmap', 'osd}', 'overrides/{client-shutdown', 'prefetch_dirfrags/yes', 'prefetch_entire_dirfrags/no', 'races', 'ranks/5', 'session_timeout', 'tasks/{1-thrash/mds', 'thrashosds-health}']