Found 160 jobs Found 6 distinct failure reasons Crash: 'wait_until_healthy' reached maximum tries (150) after waiting for 900 seconds ceph version 14.2.21-256-g1959830da7c (1959830da7cf48ac44408f4e89326eb275b7f409) nautilus (stable) 1: (()+0x12980) [0x7ff0d918f980] 2: (gsignal()+0xc7) [0x7ff0d8287fb7] 3: (abort()+0x141) [0x7ff0d8289921] 4: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1a3) [0x7ff0d9ded543] 5: (ceph::__ceph_assertf_fail(char const*, char const*, int, char const*, char const*, ...)+0) [0x7ff0d9ded6cd] 6: (Mutex::lock(bool)+0xa8) [0x7ff0d9e6db38] 7: (PyModuleRunner::log(int, std::__cxx11::basic_string, std::allocator > const&)+0x100) [0x563b955d45f0] 8: (()+0x156801) [0x563b95532801] 9: (PyEval_EvalFrameEx()+0x8010) [0x7ff0d96ae780] 10: (PyEval_EvalFrameEx()+0x8b5b) [0x7ff0d96af2cb] 11: (PyEval_EvalFrameEx()+0x8b5b) [0x7ff0d96af2cb] 12: (PyEval_EvalFrameEx()+0x8b5b) [0x7ff0d96af2cb] 13: (PyEval_EvalFrameEx()+0x8b5b) [0x7ff0d96af2cb] 14: (PyEval_EvalCodeEx()+0x7d8) [0x7ff0d97eb908] 15: (()+0x17a72d) [0x7ff0d978672d] 16: (PyObject_Call()+0x43) [0x7ff0d9780913] 17: (PyEval_EvalFrameEx()+0x5314) [0x7ff0d96aba84] 18: (PyEval_EvalCodeEx()+0x7d8) [0x7ff0d97eb908] 19: (PyEval_EvalFrameEx()+0x5bf6) [0x7ff0d96ac366] 20: (PyEval_EvalCodeEx()+0x7d8) [0x7ff0d97eb908] 21: (()+0x17a72d) [0x7ff0d978672d] 22: (PyObject_Call()+0x43) [0x7ff0d9780913] 23: (PyEval_EvalFrameEx()+0x5314) [0x7ff0d96aba84] 24: (PyEval_EvalFrameEx()+0x8b5b) [0x7ff0d96af2cb] 25: (PyEval_EvalFrameEx()+0x8b5b) [0x7ff0d96af2cb] 26: (PyEval_EvalCodeEx()+0x7d8) [0x7ff0d97eb908] 27: (()+0x17a649) [0x7ff0d9786649] 28: (PyObject_Call()+0x43) [0x7ff0d9780913] 29: (()+0x1a1eec) [0x7ff0d97adeec] 30: (PyObject_Call()+0x43) [0x7ff0d9780913] 31: (PyEval_CallObjectWithKeywords()+0x47) [0x7ff0d97eae37] 32: (()+0xeb462) [0x7ff0d96f7462] 33: (()+0x1b2ae7) [0x7ff0d97beae7] 34: (()+0x76db) [0x7ff0d91846db] 35: (clone()+0x3f) [0x7ff0d836a71f] 25 jobs: ['6132079', '6132237', '6132295', '6132229', '6132215', '6132022', '6132104', '6132332', '6132309', '6132340', '6132322', '6132141', '6132177', '6132243', '6132283', '6132135', '6132275', '6132169', '6132192', '6132096', '6132085', '6132035', '6132119', '6132267', '6132149'] suites intersection: ['1-client', '2-upgrade', 'conf/{client', 'mds', 'mon', 'osd}', 'overrides/{frag_enable', 'whitelist_health', 'whitelist_wrongly_marked_down}'] suites union: ['1-client', '2-upgrade', '3-client-upgrade', '3-compat_client/mimic}}', '3-compat_client/no}}', '3-sanity', '3-verify}', '4-client-upgrade/no', '4-client-upgrade/yes', '4-compat_client', '5-client-sanity', '5-client-sanity}}', '6-snap-upgrade', '7-client-sanity}}', 'clusters/1-mds-2-client-micro', 'conf/{client', 'fs/upgrade/featureful_client/old_client/{bluestore-bitmap', 'fs/upgrade/featureful_client/upgraded_client/{bluestore-bitmap', 'fs/upgrade/snaps/{clusters/3-mds', 'fs/upgrade/volumes/import-legacy/{bluestore-bitmap', 'mds', 'mon', 'multimds/no', 'multimds/yes', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'objectstore-ec/filestore-xfs', 'osd}', 'overrides/{frag_enable', 'pg-warn', 'tasks/{0-luminous', 'tasks/{0-mimic', 'ubuntu_18.04}', 'whitelist_health', 'whitelist_rstat', 'whitelist_wrongly_marked_down}'] Failure: 3 jobs: ['6132110', '6132210', '6132303'] suites intersection: ['centos_latest', 'clusters/fixed-2-ucephfs', 'conf/{client', 'fs/verify/{begin', 'mds', 'mon', 'mon-debug', 'mount/fuse', 'osd}', 'overrides/{frag_enable', 'session_timeout', 'tasks/cfuse_workunit_suites_fsstress', 'validater/valgrind}', 'whitelist_health', 'whitelist_wrongly_marked_down}'] suites union: ['centos_latest', 'clusters/fixed-2-ucephfs', 'conf/{client', 'fs/verify/{begin', 'mds', 'mon', 'mon-debug', 'mount/fuse', 'objectstore-ec/bluestore-bitmap', 'objectstore-ec/bluestore-comp-ec-root', 'objectstore-ec/bluestore-ec-root', 'osd}', 'overrides/{frag_enable', 'session_timeout', 'tasks/cfuse_workunit_suites_fsstress', 'validater/valgrind}', 'whitelist_health', 'whitelist_wrongly_marked_down}'] Failure: Test failure: test_async_subvolume_rm (tasks.cephfs.test_volumes.TestVolumes) 2 jobs: ['6132219', '6132047'] suites intersection: ['clusters/1-mds-4-client-coloc', 'conf/{client', 'fs/basic_functional/{begin', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile', 'osd}', 'overrides/{frag_enable', 'tasks/volumes}', 'whitelist_health', 'whitelist_wrongly_marked_down}'] suites union: ['clusters/1-mds-4-client-coloc', 'conf/{client', 'fs/basic_functional/{begin', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-ec-root', 'osd}', 'overrides/{frag_enable', 'supported-random-distros$/{centos_latest}', 'supported-random-distros$/{ubuntu_latest}', 'tasks/volumes}', 'whitelist_health', 'whitelist_wrongly_marked_down}'] Crash: Test failure: test_rebuild_simple (tasks.cephfs.test_data_scan.TestDataScan) ceph version 14.2.21-256-g1959830da7c (1959830da7cf48ac44408f4e89326eb275b7f409) nautilus (stable) 1: (()+0xf630) [0x7f4e4ffbc630] 2: (CInode::add_remote_parent(CDentry*)+0x14) [0x562f177a80d4] 3: (CDentry::pop_projected_linkage()+0x196) [0x562f17771ac6] 4: (MDCache::_create_system_file_finish(boost::intrusive_ptr&, CDentry*, unsigned long, MDSContext*)+0xfb) [0x562f1765d06b] 5: (MDSContext::complete(int)+0x74) [0x562f17815f44] 6: (MDSIOContextBase::complete(int)+0x16f) [0x562f1781619f] 7: (MDSLogContextBase::complete(int)+0x40) [0x562f178163b0] 8: (Finisher::finisher_thread_entry()+0x16f) [0x7f4e5218a51f] 9: (()+0x7ea5) [0x7f4e4ffb4ea5] 10: (clone()+0x6d) [0x7f4e4ec628dd] 1 jobs: ['6132293'] suites: ['clusters/1-mds-4-client-coloc', 'conf/{client', 'fs/basic_functional/{begin', 'mds', 'mon', 'mount/fuse', 'no_client_pidfile', 'objectstore/bluestore-bitmap', 'osd}', 'overrides/{frag_enable', 'supported-random-distros$/{centos_latest}', 'tasks/data-scan}', 'whitelist_health', 'whitelist_wrongly_marked_down}'] Failure: 'wait_until_healthy' reached maximum tries (150) after waiting for 900 seconds 2 jobs: ['6132069', '6132200'] suites intersection: ['1-client', '2-upgrade', 'clusters/1-mds-2-client-micro', 'conf/{client', 'fs/upgrade/featureful_client/old_client/{bluestore-bitmap', 'mds', 'mon', 'multimds/no', 'osd}', 'overrides/{frag_enable', 'tasks/{0-luminous', 'whitelist_health', 'whitelist_wrongly_marked_down}'] suites union: ['1-client', '2-upgrade', '3-compat_client/mimic}}', '3-compat_client/no}}', 'clusters/1-mds-2-client-micro', 'conf/{client', 'fs/upgrade/featureful_client/old_client/{bluestore-bitmap', 'mds', 'mon', 'multimds/no', 'osd}', 'overrides/{frag_enable', 'tasks/{0-luminous', 'whitelist_health', 'whitelist_wrongly_marked_down}'] Failure: "2021-05-24 22:17:59.050113 mon.a (mon.0) 123 : cluster [ERR] Health check failed: Module 'volumes' has failed: Not found or unloadable (MGR_MODULE_ERROR)" in cluster log 1 jobs: ['6132041'] suites: ['1-client', '2-upgrade', '3-sanity', '4-client-upgrade/yes', '5-client-sanity', '6-snap-upgrade', '7-client-sanity}}', 'conf/{client', 'fs/upgrade/snaps/{clusters/3-mds', 'mds', 'mon', 'multimds/no', 'objectstore-ec/filestore-xfs', 'osd}', 'overrides/{frag_enable', 'tasks/{0-luminous', 'whitelist_health', 'whitelist_rstat', 'whitelist_wrongly_marked_down}']