Found 24 jobs Found 3 distinct failure reasons Dead: b'2021-04-26T04:43:12.477 DEBUG:teuthology.orchestra.run.gibba030:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 30 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-osd.7.asok dump_ops_in_flight' 1 jobs: ['6072790'] suites: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{default}', 'backoff/peering', 'ceph', 'clusters/{fixed-2', 'crc-failures/bad_map_crc_failure', 'd-balancer/on', 'mon_election/connectivity', 'msgr-failures/osd-delay', 'msgr/async-v1only', 'objectstore/bluestore-stupid', 'openstack}', 'rados', 'rados/thrash/{0-size-min-size-overrides/2-size-2-min-size', 'supported-random-distro$/{rhel_8}', 'thrashers/morepggrow', 'thrashosds-health', 'workloads/cache-snaps}'] Crash: Found coredumps on ubuntu@gibba018.front.sepia.ceph.com ceph version 17.0.0-3041-g8a50001f (8a50001f30abbbde31423840fc60d0ef25994b3a) quincy (dev) 1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x153c0) [0x7f92fdc093c0] 2: gsignal() 3: abort() 4: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1a7) [0x557de8f9daf8] 5: ceph-osd(+0xbfdc96) [0x557de8f9dc96] 6: (ScrubQueue::update_scrub_job(boost::intrusive_ptr, ScrubQueue::sched_params_t const&)+0x52b) [0x557de92cc99b] 7: (PgScrubber::update_scrub_job(requested_scrub_t const&)+0x2b3) [0x557de92ae523] 8: (PgScrubber::on_primary_change(requested_scrub_t const&)+0xb2) [0x557de92ae622] 9: (PG::on_info_history_change()+0x3b) [0x557de9130a9b] 10: (PeeringState::update_stats(std::function, ceph::os::Transaction*)+0x3d) [0x557de937406d] 11: (PrimaryLogPG::agent_choose_mode(bool, boost::intrusive_ptr)+0x5ae) [0x557de91d2ade] 12: (PrimaryLogPG::finish_promote(int, PrimaryLogPG::CopyResults*, std::shared_ptr)+0xa2a) [0x557de91d469a] 13: (PromoteCallback::finish(boost::tuples::tuple)+0x78) [0x557de9274108] 14: (PrimaryLogPG::process_copy_chunk(hobject_t, unsigned long, int)+0x716) [0x557de91e5d86] 15: (C_Copyfrom::finish(int)+0x15c) [0x557de927ad9c] 16: (Context::complete(int)+0xd) [0x557de90b392d] 17: (Finisher::finisher_thread_entry()+0x19d) [0x557de979044d] 18: (Thread::_entry_func(void*)+0xd) [0x557de97b06cd] 19: /lib/x86_64-linux-gnu/libpthread.so.0(+0x9609) [0x7f92fdbfd609] 20: clone() 1 jobs: ['6072786'] suites: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{more-async-partial-recovery}', 'backoff/normal', 'ceph', 'clusters/{fixed-2', 'crc-failures/bad_map_crc_failure', 'd-balancer/on', 'mon_election/connectivity', 'msgr-failures/few', 'msgr/async', 'objectstore/bluestore-comp-zlib', 'openstack}', 'rados', 'rados/thrash/{0-size-min-size-overrides/2-size-2-min-size', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/pggrow', 'thrashosds-health', 'workloads/cache-agent-small}'] Failure: Exiting scrub checking -- not all pgs scrubbed. 1 jobs: ['6072789'] suites: ['1-pg-log-overrides/short_pg_log', '2-recovery-overrides/{more-async-recovery}', 'backoff/normal', 'ceph', 'clusters/{fixed-2', 'crc-failures/default', 'd-balancer/crush-compat', 'mon_election/classic', 'msgr-failures/few', 'msgr/async', 'objectstore/bluestore-low-osd-mem-target', 'openstack}', 'rados', 'rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'supported-random-distro$/{rhel_8}', 'thrashers/mapgap', 'thrashosds-health', 'workloads/cache-snaps-balanced}']