Found 173 jobs Found 7 distinct failure reasons Crash: "2023-11-09T23:56:16.972202+0000 mon.a (mon.0) 233 : cluster [WRN] Replacing daemon mds.a as rank 0 with standby daemon mds.b" in cluster log ceph version 16.2.14-147-g4c25bdc7 (4c25bdc7f0551713156708a897fdcab125232b7c) pacific (stable) 1: /lib64/libpthread.so.0(+0x12b20) [0x7fe78932cb20] 2: gsignal() 3: abort() 4: /lib64/libstdc++.so.6(+0x9009b) [0x7fe78872d09b] 5: /lib64/libstdc++.so.6(+0x9653c) [0x7fe78873353c] 6: /lib64/libstdc++.so.6(+0x96597) [0x7fe788733597] 7: /lib64/libstdc++.so.6(+0x967f8) [0x7fe7887337f8] 8: ceph-mds(+0x2145a3) [0x55648992b5a3] 9: (Option::size_t const md_config_t::get_val(ConfigValues const&, std::basic_string_view >) const+0x5c) [0x5564899a0bfc] 10: (SessionMap::SessionMap(MDSRank*)+0x2c2) [0x556489be99f2] 11: (MDSRank::MDSRank(int, std::__cxx11::basic_string, std::allocator >, ceph::fair_mutex&, std::shared_ptr&, CommonSafeTimer&, Beacon&, std::unique_ptr >&, Messenger*, MonClient*, MgrClient*, Context*, Context*, boost::asio::io_context&)+0x214) [0x556489904954] 12: (MDSRankDispatcher::MDSRankDispatcher(int, std::__cxx11::basic_string, std::allocator >, ceph::fair_mutex&, std::shared_ptr&, CommonSafeTimer&, Beacon&, std::unique_ptr >&, Messenger*, MonClient*, MgrClient*, Context*, Context*, boost::asio::io_context&)+0x101) [0x5564899063e1] 13: (MDSDaemon::handle_mds_map(boost::intrusive_ptr const&)+0x1608) [0x5564898d6368] 14: (MDSDaemon::handle_core_message(boost::intrusive_ptr const&)+0xcd) [0x5564898d946d] 15: (MDSDaemon::ms_dispatch2(boost::intrusive_ptr const&)+0xc3) [0x5564898d9fc3] 16: (DispatchQueue::entry()+0x126a) [0x7fe78a586e0a] 17: (DispatchQueue::DispatchThread::entry()+0x11) [0x7fe78a63b5b1] 18: /lib64/libpthread.so.0(+0x814a) [0x7fe78932214a] 134 jobs suites intersection: [] suites union: ['2-three-per-cluster}', '2-workunit/cfuse_workunit_snaptests}}', '2-workunit/cfuse_workunit_suites_fsstress}}', '2-workunit/cfuse_workunit_suites_pjd}}', '2-workunit/cfuse_workunit_trivial_sync}}', '2-workunit/ffsb}}', '2-workunit/fs/snaps}}', '2-workunit/fs/trivial_sync}}', '2-workunit/iozone}}', '2-workunit/suites/ffsb}}', '2-workunit/suites/fsstress}}', '2-workunit/suites/iozone}}', '2-workunit/suites/pjd}}', 'centos_latest', 'cephfs-mirror/one-per-cluster', 'cephfs-mirror/{1-volume-create-rm', 'clients/mirror', 'clients/{mirror}', 'cluster/1-node', 'cluster/{1-node}', 'clusters/1-mds-1-client-coloc', 'clusters/1-mds-2-client', 'clusters/1-mds-3-client', 'clusters/1-node-1-mds-1-osd', 'clusters/1a11s-mds-1c-client-3node', 'clusters/1a3s-mds-1c-client', 'clusters/1a3s-mds-2c-client', 'clusters/1a3s-mds-4c-client', 'clusters/1a5s-mds-1c-client', 'clusters/fixed-2-ucephfs', 'clusters/small-cluster', 'conf/{client', 'debug', 'distro/{centos_8}', 'distro/{rhel_8}', 'distro/{ubuntu_latest}', 'distros/ubuntu_latest', 'fs/32bits/{begin', 'fs/bugs/client_trim_caps/{begin', 'fs/full/{begin', 'fs/functional/{begin', 'fs/libcephfs/{begin', 'fs/mirror-ha/{begin', 'fs/mirror/{begin', 'fs/mixed-clients/{begin', 'fs/multiclient/{begin', 'fs/multifs/{begin', 'fs/permission/{begin', 'fs/shell/{begin', 'fs/snaps/{begin', 'fs/thrash/multifs/{begin', 'fs/thrash/workloads/{begin', 'fs/top/{begin', 'fs/traceless/{begin', 'fs/valgrind/{begin', 'fs/verify/{begin', 'fs/volumes/{begin', 'fs/workload/{begin', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'ignorelist_wrongly_marked_down}', 'k-testing}', 'kclient-overrides/{distro/stock/{k-stock', 'kclient-overrides/{distro/testing/{flavor/centos_latest', 'kclient-overrides/{distro/testing/{flavor/ubuntu_latest', 'mds', 'mirror/{cephfs-mirror/one-per-cluster', 'mon', 'mon-debug', 'mon-debug}', 'mount', 'mount/fuse', 'mount/kclient/{k-testing', 'mount/kclient/{mount', 'ms-die-on-skipped}', 'ms-die-on-skipped}}', 'ms_mode/{crc}', 'ms_mode/{legacy}', 'ms_mode/{secure}', 'msgr-failures/none', 'msgr-failures/osd-mds-delay', 'multifs', 'no_client_pidfile}', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-ec-root', 'omap_limit/10', 'omap_limit/10000', 'osd-asserts', 'osd-asserts}', 'osd}', 'overrides', 'overrides/ignorelist_health', 'overrides/whitelist_health', 'overrides/{distro/stock/{k-stock', 'overrides/{distro/testing/{flavor/centos_latest', 'overrides/{distro/testing/{flavor/ubuntu_latest', 'overrides/{faked-ino', 'overrides/{frag', 'overrides/{ignorelist_health', 'overrides/{whitelist_health}', 'prefetch_dirfrags/no', 'prefetch_dirfrags/yes', 'races', 'ranks/1', 'ranks/3', 'ranks/5', 'rhel_8}', 'scrub/no', 'scrub/yes', 'session_timeout', 'session_timeout}', 'standby-replay', 'subvolume/{no-subvolume}', 'subvolume/{with-namespace-isolated-and-quota}', 'subvolume/{with-namespace-isolated}', 'subvolume/{with-no-extra-options}', 'subvolume/{with-quota}', 'supported-random-distro$/{rhel_8}', 'supported-random-distros$/{centos_8}', 'supported-random-distros$/{rhel_8}', 'tasks/acls}', 'tasks/admin}', 'tasks/alternate-pool}', 'tasks/asok_dump_tree}', 'tasks/auto-repair}', 'tasks/backtrace}', 'tasks/cephfs-shell}', 'tasks/cephfs_misc_tests}', 'tasks/cfuse_workunit_misc}', 'tasks/cfuse_workunit_suites_blogbench', 'tasks/cfuse_workunit_suites_dbench', 'tasks/cfuse_workunit_suites_ffsb', 'tasks/cfuse_workunit_suites_fsstress', 'tasks/cfuse_workunit_suites_fsstress}', 'tasks/cfuse_workunit_suites_pjd}', 'tasks/client-limits}', 'tasks/client-readahead}', 'tasks/client-recovery}', 'tasks/client}', 'tasks/damage}', 'tasks/data-scan}', 'tasks/dbench', 'tasks/exports}', 'tasks/failover}', 'tasks/forward-scrub}', 'tasks/fragment}', 'tasks/fsstress', 'tasks/fstop}', 'tasks/ino_release_cb}', 'tasks/ior-shared-file}', 'tasks/journal-repair}', 'tasks/kernel_cfuse_workunits_dbench_iozone}', 'tasks/kernel_cfuse_workunits_untarbuild_blogbench}', 'tasks/libcephfs/{frag', 'tasks/libcephfs_python}', 'tasks/mds-flush}', 'tasks/mds-full}', 'tasks/mds_creation_retry}', 'tasks/mdtest}', 'tasks/metrics}', 'tasks/mgr-osd-full}', 'tasks/mirror}', 'tasks/mirror}}', 'tasks/multifs-auth}', 'tasks/multimds_misc}', 'tasks/openfiletable}', 'tasks/pool-perm}', 'tasks/quota}', 'tasks/recovery-fs}', 'tasks/scrub}', 'tasks/sessionmap}', 'tasks/snap-schedule}', 'tasks/snap_schedule_snapdir}', 'tasks/snapshots}', 'tasks/strays}', 'tasks/test_journal_migration}', 'tasks/trim-i24137}', 'tasks/truncate_delay}', 'tasks/volume-client}', 'tasks/volumes/{overrides', 'tasks/workunit/quota}', 'tasks/workunit/snaps}', 'tasks/{0-check-counter', 'tasks/{1-thrash/mds', 'tasks/{1-thrash/mon', 'tasks/{1-thrash/osd', 'test/basic}}', 'test/clone}}', 'test/finisher_per_module}}', 'test/misc}}', 'test/snapshot}}', 'test}}', 'thrashosds-health}', 'traceless/50pc}', 'validater/lockdep}', 'validater/valgrind}', 'workloads/cephfs-mirror-ha-workunit}', 'workunit/direct_io}', 'workunit/fs/misc}', 'workunit/fs/norstats}', 'workunit/fs/test_o_trunc}', 'workunit/kernel_untar_build}', 'workunit/suites/dbench}', 'workunit/suites/ffsb}', 'workunit/suites/fsstress}', 'workunit/suites/fsx}', 'workunit/suites/fsync-tester}', 'workunit/suites/iogen}', 'workunit/suites/iozone}', 'wsync/{no}}', 'wsync/{yes}}'] Crash: "2023-11-09T22:46:02.923104+0000 mon.a (mon.0) 232 : cluster [WRN] Replacing daemon mds.b as rank 0 with standby daemon mds.a" in cluster log ceph version 16.2.14-147-g4c25bdc7 (4c25bdc7f0551713156708a897fdcab125232b7c) pacific (stable) 1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7fe1fee10420] 2: gsignal() 3: abort() 4: /lib/x86_64-linux-gnu/libstdc++.so.6(+0x9e911) [0x7fe1f5ebc911] 5: /lib/x86_64-linux-gnu/libstdc++.so.6(+0xaa38c) [0x7fe1f5ec838c] 6: /lib/x86_64-linux-gnu/libstdc++.so.6(+0xaa3f7) [0x7fe1f5ec83f7] 7: /lib/x86_64-linux-gnu/libstdc++.so.6(+0xaa6a9) [0x7fe1f5ec86a9] 8: ceph-mds(+0x18633d) [0x558f22d6433d] 9: ceph-mds(+0x186963) [0x558f22d64963] 10: (MDSRank::MDSRank(int, std::__cxx11::basic_string, std::allocator >, ceph::fair_mutex&, std::shared_ptr&, CommonSafeTimer&, Beacon&, std::unique_ptr >&, Messenger*, MonClient*, MgrClient*, Context*, Context*, boost::asio::io_context&)+0x20f) [0x558f22dd61ef] 11: (MDSRankDispatcher::MDSRankDispatcher(int, std::__cxx11::basic_string, std::allocator >, ceph::fair_mutex&, std::shared_ptr&, CommonSafeTimer&, Beacon&, std::unique_ptr >&, Messenger*, MonClient*, MgrClient*, Context*, Context*, boost::asio::io_context&)+0x115) [0x558f22dd7c25] 12: (MDSDaemon::handle_mds_map(boost::intrusive_ptr const&)+0x1530) [0x558f22da2ae0] 13: (MDSDaemon::handle_core_message(boost::intrusive_ptr const&)+0x3df) [0x558f22da6a8f] 14: (MDSDaemon::ms_dispatch2(boost::intrusive_ptr const&)+0x186) [0x558f22da7426] 15: (Messenger::ms_deliver_dispatch(boost::intrusive_ptr const&)+0x460) [0x7fe1f64bca30] 16: (DispatchQueue::entry()+0x58f) [0x7fe1f64ba2cf] 17: (DispatchQueue::DispatchThread::entry()+0x11) [0x7fe1f6589081] 18: /lib/x86_64-linux-gnu/libpthread.so.0(+0x8609) [0x7fe1fee04609] 17 jobs: ['7453215', '7453232', '7453152', '7453183', '7453342', '7453315', '7453217', '7453229', '7453323', '7453165', '7453244', '7453362', '7453239', '7453295', '7453273', '7453261', '7453289'] suites intersection: ['conf/{client', 'mds', 'mon', 'osd}'] suites union: ['2-workunit/cfuse_workunit_suites_pjd}}', '2-workunit/ffsb}}', '2-workunit/fs/trivial_sync}}', '2-workunit/iozone}}', '2-workunit/suites/pjd}}', 'clusters/1-mds-2-client', 'clusters/1a11s-mds-1c-client-3node', 'clusters/1a3s-mds-1c-client', 'clusters/1a3s-mds-2c-client', 'clusters/1a3s-mds-4c-client', 'clusters/1a5s-mds-1c-client', 'clusters/fixed-2-ucephfs', 'conf/{client', 'distro/{centos_8}', 'distro/{rhel_8}', 'distro/{ubuntu/{latest', 'distro/{ubuntu_latest}', 'distros/ubuntu_latest', 'fs/functional/{begin', 'fs/multiclient/{begin', 'fs/multifs/{begin', 'fs/snaps/{begin', 'fs/thrash/multifs/{begin', 'fs/thrash/workloads/{begin', 'fs/traceless/{begin', 'fs/verify/{begin', 'fs/workload/{begin', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'ignorelist_wrongly_marked_down}', 'k-testing}', 'mds', 'mon', 'mon-debug', 'mon-debug}', 'mount', 'mount/fuse', 'mount/kclient/{k-testing', 'mount/kclient/{mount', 'ms-die-on-skipped}', 'ms-die-on-skipped}}', 'ms_mode/{crc}', 'ms_mode/{secure}', 'msgr-failures/none', 'msgr-failures/osd-mds-delay', 'multifs', 'no_client_pidfile}', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'objectstore/bluestore-bitmap', 'omap_limit/10', 'osd-asserts', 'osd}', 'overrides/{distro/stock/{k-stock', 'overrides/{distro/testing/{flavor/centos_latest', 'overrides/{distro/testing/{flavor/ubuntu_latest', 'overrides/{frag', 'overrides/{ignorelist_health', 'overrides}}', 'prefetch_dirfrags/yes', 'races', 'ranks/1', 'ranks/3', 'ranks/5', 'rhel_8}', 'scrub/yes', 'session_timeout', 'session_timeout}', 'standby-replay', 'subvolume/{with-namespace-isolated-and-quota}', 'subvolume/{with-namespace-isolated}', 'tasks/cap-flush}', 'tasks/cephfs_misc_tests}', 'tasks/cfuse_workunit_suites_fsstress', 'tasks/fsstress', 'tasks/multifs-auth}', 'tasks/openfiletable}', 'tasks/pool-perm}', 'tasks/workunit/dir-max-entries}', 'tasks/workunit/snaps}', 'tasks/{0-check-counter', 'tasks/{1-thrash/mon', 'tasks/{1-thrash/osd', 'thrashosds-health}', 'traceless/50pc}', 'validater/lockdep}', 'workunit/suites/blogbench}', 'workunit/suites/pjd}', 'wsync/{yes}}'] Dead: b'2023-11-10T10:35:57.230 DEBUG:teuthology.exit:Finished running handlers' 14 jobs: ['7453218', '7453161', '7453150', '7453187', '7453170', '7453209', '7453255', '7453364', '7453181', '7453348', '7453265', '7453327', '7453291', '7453301'] suites intersection: ['conf/{client', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn}'] suites union: ['1-client', '1-ranks/1', '1-ranks/2', '1-volume/{0-create', '2-allow_standby_replay/no', '2-allow_standby_replay/yes', '2-client', '2-upgrade', '3-client-upgrade', '3-compat_client/no}}', '3-compat_client/pacific}}', '3-inline/no', '3-inline/yes', '3-upgrade-with-workload', '4-compat_client', '4-verify}', '4-verify}}', '5-client-sanity}}', 'centos_8.stream_container_tools', 'centos_latest', 'clusters/1-mds-2-client-micro', 'conf/{client', 'fs/upgrade/featureful_client/old_client/{bluestore-bitmap', 'fs/upgrade/featureful_client/upgraded_client/{bluestore-bitmap', 'fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'multimds/no', 'multimds/yes', 'osd}', 'overrides/{ignorelist_health', 'pg-warn}', 'roles', 'tasks/{0-from/v16.2.4', 'tasks/{0-nautilus'] Failure: SSH connection to smithi196 was lost: 'sudo yum install -y kernel' 3 jobs: ['7453148', '7453178', '7453177'] suites intersection: ['conf/{client', 'distro/{centos_8}', 'mds', 'mon', 'mount/fuse', 'osd}'] suites union: ['clusters/1a11s-mds-1c-client-3node', 'clusters/fixed-2-ucephfs', 'conf/{client', 'distro/{centos_8}', 'fs/32bits/{begin', 'fs/permission/{begin', 'fs/workload/{begin', 'ignorelist_health', 'ignorelist_wrongly_marked_down', 'ignorelist_wrongly_marked_down}', 'mds', 'mon', 'mount/fuse', 'ms_mode/{crc}', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-ec-root', 'omap_limit/10000', 'osd-asserts', 'osd}', 'overrides/{faked-ino', 'overrides/{frag', 'overrides/{ignorelist_health', 'ranks/1', 'scrub/no', 'session_timeout}', 'standby-replay', 'subvolume/{no-subvolume}', 'tasks/cfuse_workunit_misc}', 'tasks/cfuse_workunit_suites_fsstress}', 'tasks/{0-check-counter', 'workunit/fs/norstats}', 'wsync/{yes}}'] Failure: timeout expired in wait_until_healthy 1 jobs: ['7453245'] suites: ['1-client', '2-upgrade', '3-verify}', 'clusters/1-mds-2-client-micro', 'conf/{client', 'fs/upgrade/volumes/import-legacy/{bluestore-bitmap', 'ignorelist_wrongly_marked_down', 'mds', 'mon', 'osd}', 'overrides/{ignorelist_health', 'pg-warn}', 'tasks/{0-nautilus', 'ubuntu_18.04}'] Failure: Timed out waiting for MDS daemons to become healthy 1 jobs: ['7453196'] suites: ['1-mount', '2-workload/dbench', 'distro/centos_latest}', 'fs/cephadm/multivolume/{0-start'] Failure: Command failed on smithi196 with status 1: 'sudo rpm -ivh --oldpackage --replacefiles --replacepkgs /tmp/kernel.x86_64.rpm' 1 jobs: ['7453149'] suites: ['clusters/1a3s-mds-4c-client', 'conf/{client', 'distro/{ubuntu_latest}', 'fs/functional/{begin', 'ignorelist_wrongly_marked_down', 'k-testing}', 'mds', 'mon', 'mount/kclient/{mount', 'ms-die-on-skipped}}', 'no_client_pidfile}', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{distro/testing/{flavor/centos_latest', 'overrides/{ignorelist_health', 'tasks/multimds_misc}']