Found 220 jobs Found 3 distinct failure reasons Failure: Test failure: test_a_set_login_credentials (tasks.mgr.dashboard.test_auth.AuthTest) 3 jobs: ['5920840', '5920950', '5921023'] suites intersection: ['debug/mgr', 'rados/dashboard/{clusters/{2-node-mgr}', 'supported-random-distro$/{ubuntu_latest}', 'tasks/dashboard}'] suites union: ['debug/mgr', 'mon_election/classic', 'mon_election/connectivity', 'objectstore/bluestore-comp-lz4', 'objectstore/bluestore-hybrid', 'objectstore/filestore-xfs', 'rados/dashboard/{clusters/{2-node-mgr}', 'supported-random-distro$/{ubuntu_latest}', 'tasks/dashboard}'] Crash: "2021-02-28T21:43:43.820597+0000 mon.a (mon.0) 1781 : cluster [WRN] Health check failed: 1 daemons have recently crashed (RECENT_CRASH)" in cluster log ceph version 17.0.0-1309-g78de94bf (78de94bfdea3d61ba4985af6a0279d22372fcb4a) quincy (dev) 1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x12980) [0x7f6931a07980] 2: (std::_Hashtable >, mempool::pool_allocator<(mempool::pool_index_t)4, std::pair > >, std::__detail::_Select1st, std::equal_to, std::hash, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits >::erase(std::__detail::_Node_const_iterator >, false, true>)+0x23) [0x561127c4e1c3] 3: (BlueStore::Collection::split_cache(BlueStore::Collection*)+0x1b2) [0x561127bad0d2] 4: (BlueStore::_split_collection(BlueStore::TransContext*, boost::intrusive_ptr&, boost::intrusive_ptr&, unsigned int, int)+0x161) [0x561127bee771] 5: (BlueStore::_txc_add_transaction(BlueStore::TransContext*, ceph::os::Transaction*)+0x880) [0x561127c1f460] 6: (BlueStore::queue_transactions(boost::intrusive_ptr&, std::vector >&, boost::intrusive_ptr, ThreadPool::TPHandle*)+0x2fa) [0x561127c21dda] 7: (ObjectStore::queue_transaction(boost::intrusive_ptr&, ceph::os::Transaction&&, boost::intrusive_ptr, ThreadPool::TPHandle*)+0x80) [0x5611277285e0] 8: (OSD::dispatch_context(PeeringCtx&, PG*, std::shared_ptr, ThreadPool::TPHandle*)+0x1db) [0x5611276c31db] 9: (OSD::dequeue_peering_evt(OSDShard*, PG*, std::shared_ptr, ThreadPool::TPHandle&)+0x1e5) [0x5611276f3155] 10: (ceph::osd::scheduler::PGPeeringItem::run(OSD*, OSDShard*, boost::intrusive_ptr&, ThreadPool::TPHandle&)+0x50) [0x561127929580] 11: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0xcd5) [0x5611276e4e45] 12: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x4ac) [0x561127d59d6c] 13: (ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0x561127d5d020] 14: /lib/x86_64-linux-gnu/libpthread.so.0(+0x76db) [0x7f69319fc6db] 15: clone() 1 jobs: ['5920965'] suites: ['1-pg-log-overrides/short_pg_log', '2-recovery-overrides/{more-async-partial-recovery}', 'backoff/normal', 'ceph', 'clusters/{fixed-2', 'crc-failures/default', 'd-balancer/on', 'mon_election/connectivity', 'msgr-failures/fastclose', 'msgr/async-v1only', 'objectstore/bluestore-bitmap', 'openstack}', 'rados', 'rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/morepggrow', 'thrashosds-health', 'workloads/small-objects-localized}'] Dead: b'2021-03-01T06:38:04.427 DEBUG:teuthology.exit:Finished running handlers' 2 jobs: ['5920954', '5920865'] suites intersection: ['clusters/{fixed-2', 'mon_election/connectivity', 'msgr-failures/fastclose', 'openstack}', 'rados', 'supported-random-distro$/{ubuntu_latest}', 'thrashosds-health'] suites union: ['1-pg-log-overrides/short_pg_log', '2-recovery-overrides/{default}', 'backoff/normal', 'ceph', 'clusters/{fixed-2', 'crc-failures/default', 'd-balancer/on', 'fast/normal', 'mon_election/connectivity', 'msgr-failures/fastclose', 'msgr/async-v1only', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp-zlib', 'openstack}', 'rados', 'rados/thrash-erasure-code/{ceph', 'rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'recovery-overrides/{more-async-recovery}', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/minsize_recovery', 'thrashers/morepggrow', 'thrashosds-health', 'workloads/ec-radosbench}', 'workloads/set-chunks-read}']