Found 181 jobs
Ignoring malparsed backtrace:  ceph version 17.0.0-8454-ged9d0b44 (ed9d0b441738e5d5fee0b1394a629b504365ab69) quincy (dev)
,  1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x152) [0x5ff402c]
,  2: /usr/lib64/ceph/libceph-common.so.2(+0x28524d) [0x5ff424d]

Ignoring malparsed backtrace: ==30436==    by 0x4F00FB0: librados::v14_2_0::RadosClient::connect() (RadosClient.cc:312)
, ==30436==    by 0xFAA207: connect_cluster_pp(librados::v14_2_0::Rados&, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > const&) (test_cxx.cc:161)
, ==30436==    by 0xFAADFF: connect_cluster_pp[abi:cxx11](librados::v14_2_0::Rados&) (test_cxx.cc:126)

Ignoring malparsed backtrace: ==30436==    by 0x938CE24: ??? (in /usr/lib64/libnl-route-3.so.200.26.0)
, ==30436==    by 0x400F8B9: call_init.part.0 (in /usr/lib64/ld-2.28.so)
, ==30436==    by 0x400F9B9: _dl_init (in /usr/lib64/ld-2.28.so)

Ignoring malparsed backtrace: ==30436==    by 0xFCB676: Run (gtest.cc:2848)
, ==30436==    by 0xFCB676: testing::TestInfo::Run() (gtest.cc:2831)
, ==30436==    by 0xFCBC70: Run (gtest.cc:3012)

Ignoring malparsed backtrace: ==30436==    by 0x92C60E: librbd::ImageCtx::ImageCtx(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, char const*, librados::v14_2_0::IoCtx&, bool) (ImageCtx.cc:123)
, ==30436==    by 0x81F4BE: UnknownInlinedFun (ImageCtx.h:239)
, ==30436==    by 0x81F4BE: rbd::mirror::image_replayer::OpenImageRequest<librbd::ImageCtx>::send_open_image() (OpenImageRequest.cc:41)

Ignoring malparsed backtrace: ==30436==    by 0x6064F9A: KeyRing::set_modifier(char const*, char const*, EntityName&, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> > >&) (KeyRing.cc:106)
, ==30436==    by 0x60655DF: KeyRing::decode(ceph::buffer::v15_2_0::list::iterator_impl<true>&) (KeyRing.cc:193)
, ==30436==    by 0x6066474: KeyRing::load(ceph::common::CephContext*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) (KeyRing.cc:219)

Ignoring malparsed backtrace: ==30436==    by 0x6064BA2: encode (encoding.h:213)
, ==30436==    by 0x6064BA2: encode (encoding.h:219)
, ==30436==    by 0x6064BA2: KeyRing::set_modifier(char const*, char const*, EntityName&, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> > >&) (KeyRing.cc:117)

Ignoring malparsed backtrace: 2021-10-27T19:33:02.148 INFO:teuthology.orchestra.run.gibba016.stdout:{"epoch":1328,"fsid":"5a830528-8f3d-4eb2-add7-9b2182a4cd3c","created":"2021-10-27T19:09:35.021117+0000","modified":"2021-10-27T19:33:00.308113+0000","last_up_change":"2021-10-27T19:09:54.312852+0000","last_in_change":"2021-10-27T19:09:35.756621+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":184,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2021-10-27T19:09:54.736136+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2021-10-27T19:09:56.157326+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":147,"pool_name":"test-rbd-mirror-gibba016-30436-145","create_time":"2021-10-27T19:27:57.881233+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1029","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":148,"pool_name":"test-rbd-mirror-gibba016-30436-146","create_time":"2021-10-27T19:27:59.285331+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1031","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":183,"pool_name":"test-rados-api-gibba016-30436-181","create_time":"2021-10-27T19:32:39.980131+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1326","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":8,"snap_epoch":1326,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":184,"pool_name":"test-rados-api-gibba016-30436-182","create_time":"2021-10-27T19:32:42.015126+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1327","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":10,"snap_epoch":1327,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"a18d1ceb-edb8-41f1-803d-f4480b2941ed","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":1311,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6802","nonce":28946},{"type":"v1","addr":"172.21.2.116:6804","nonce":28946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6806","nonce":28946},{"type":"v1","addr":"172.21.2.116:6808","nonce":28946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6814","nonce":28946},{"type":"v1","addr":"172.21.2.116:6815","nonce":28946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6810","nonce":28946},{"type":"v1","addr":"172.21.2.116:6813","nonce":28946}]},"public_addr":"172.21.2.116:6804/28946","cluster_addr":"172.21.2.116:6808/28946","heartbeat_back_addr":"172.21.2.116:6815/28946","heartbeat_front_addr":"172.21.2.116:6813/28946","state":["exists","up"]},{"osd":1,"uuid":"7545bf9e-4f37-4aa5-b2dc-d5994995e31c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":1311,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6800","nonce":28947},{"type":"v1","addr":"172.21.2.116:6801","nonce":28947}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6803","nonce":28947},{"type":"v1","addr":"172.21.2.116:6805","nonce":28947}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6811","nonce":28947},{"type":"v1","addr":"172.21.2.116:6812","nonce":28947}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6807","nonce":28947},{"type":"v1","addr":"172.21.2.116:6809","nonce":28947}]},"public_addr":"172.21.2.116:6801/28947","cluster_addr":"172.21.2.116:6805/28947","heartbeat_back_addr":"172.21.2.116:6812/28947","heartbeat_front_addr":"172.21.2.116:6809/28947","state":["exists","up"]},{"osd":2,"uuid":"c1f6b3b1-c86b-449f-aacf-d757769f5caf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":1311,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6816","nonce":28948},{"type":"v1","addr":"172.21.2.116:6817","nonce":28948}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6818","nonce":28948},{"type":"v1","addr":"172.21.2.116:6819","nonce":28948}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6822","nonce":28948},{"type":"v1","addr":"172.21.2.116:6823","nonce":28948}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6820","nonce":28948},{"type":"v1","addr":"172.21.2.116:6821","nonce":28948}]},"public_addr":"172.21.2.116:6817/28948","cluster_addr":"172.21.2.116:6819/28948","heartbeat_back_addr":"172.21.2.116:6823/28948","heartbeat_front_addr":"172.21.2.116:6821/28948","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-10-27T19:09:36.834954+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-10-27T19:09:36.848998+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-10-27T19:09:36.848327+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"147.5","mappings":[{"from":2,"to":0}]},{"pgid":"147.1b","mappings":[{"from":0,"to":1}]},{"pgid":"147.1e","mappings":[{"from":2,"to":1}]}],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":24,"snaps":[{"begin":2,"length":1}]},{"pool":25,"snaps":[{"begin":4,"length":2},{"begin":8,"length":2},{"begin":12,"length":1}]},{"pool":29,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":30,"snaps":[{"begin":3,"length":1},{"begin":5,"length":1}]},{"pool":31,"snaps":[{"begin":2,"length":1}]},{"pool":32,"snaps":[{"begin":3,"length":1}]},{"pool":34,"snaps":[{"begin":3,"length":1}]},{"pool":36,"snaps":[{"begin":3,"length":1}]},{"pool":38,"snaps":[{"begin":3,"length":1}]},{"pool":39,"snaps":[{"begin":2,"length":1}]},{"pool":40,"snaps":[{"begin":3,"length":1}]},{"pool":41,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":42,"snaps":[{"begin":5,"length":1}]},{"pool":43,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":44,"snaps":[{"begin":5,"length":1}]},{"pool":45,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":46,"snaps":[{"begin":5,"length":1}]},{"pool":47,"snaps":[{"begin":4,"length":1},{"begin":6,"length":1}]},{"pool":48,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":49,"snaps":[{"begin":4,"length":1},{"begin":6,"length":1}]},{"pool":50,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":51,"snaps":[{"begin":6,"length":1}]},{"pool":52,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":53,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":54,"snaps":[{"begin":5,"length":1}]},{"pool":55,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":56,"snaps":[{"begin":5,"length":1}]},{"pool":57,"snaps":[{"begin":5,"length":1}]},{"pool":58,"snaps":[{"begin":6,"length":1}]},{"pool":59,"snaps":[{"begin":2,"length":1},{"begin":5,"length":1}]},{"pool":60,"snaps":[{"begin":6,"length":1}]},{"pool":61,"snaps":[{"begin":4,"length":2}]},{"pool":62,"snaps":[{"begin":5,"length":2}]},{"pool":63,"snaps":[{"begin":5,"length":1}]},{"pool":64,"snaps":[{"begin":6,"length":1}]},{"pool":65,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":66,"snaps":[{"begin":5,"length":1}]},{"pool":69,"snaps":[{"begin":2,"length":1}]},{"pool":70,"snaps":[{"begin":3,"length":1}]},{"pool":71,"snaps":[{"begin":2,"length":1}]},{"pool":72,"snaps":[{"begin":3,"length":1}]},{"pool":74,"snaps":[{"begin":3,"length":1},{"begin":5,"length":1}]},{"pool":76,"snaps":[{"begin":3,"length":1}]},{"pool":78,"snaps":[{"begin":3,"length":1}]},{"pool":79,"snaps":[{"begin":2,"length":1}]},{"pool":80,"snaps":[{"begin":3,"length":1}]},{"pool":81,"snaps":[{"begin":2,"length":1}]},{"pool":82,"snaps":[{"begin":3,"length":1}]},{"pool":83,"snaps":[{"begin":2,"length":1}]},{"pool":84,"snaps":[{"begin":5,"length":1}]},{"pool":85,"snaps":[{"begin":4,"length":1}]},{"pool":86,"snaps":[{"begin":5,"length":2}]},{"pool":87,"snaps":[{"begin":4,"length":1}]},{"pool":88,"snaps":[{"begin":5,"length":2}]},{"pool":89,"snaps":[{"begin":4,"length":1}]},{"pool":90,"snaps":[{"begin":5,"length":2}]},{"pool":91,"snaps":[{"begin":5,"length":2}]},{"pool":92,"snaps":[{"begin":5,"length":2}]},{"pool":93,"snaps":[{"begin":2,"length":1}]},{"pool":94,"snaps":[{"begin":5,"length":1}]},{"pool":96,"snaps":[{"begin":3,"length":1}]},{"pool":97,"snaps":[{"begin":4,"length":1}]},{"pool":98,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":99,"snaps":[{"begin":4,"length":1}]},{"pool":100,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":101,"snaps":[{"begin":4,"length":2}]},{"pool":102,"snaps":[{"begin":5,"length":3}]},{"pool":103,"snaps":[{"begin":4,"length":1},{"begin":6,"length":1}]},{"pool":104,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":105,"snaps":[{"begin":4,"length":1}]},{"pool":106,"snaps":[{"begin":5,"length":2}]},{"pool":109,"snaps":[{"begin":2,"length":1}]},{"pool":110,"snaps":[{"begin":3,"length":1}]},{"pool":111,"snaps":[{"begin":2,"length":1}]},{"pool":112,"snaps":[{"begin":3,"length":1}]},{"pool":114,"snaps":[{"begin":3,"length":1},{"begin":5,"length":1}]},{"pool":116,"snaps":[{"begin":3,"length":1}]},{"pool":118,"snaps":[{"begin":3,"length":1}]},{"pool":119,"snaps":[{"begin":2,"length":1}]},{"pool":120,"snaps":[{"begin":3,"length":1}]},{"pool":121,"snaps":[{"begin":2,"length":1}]},{"pool":122,"snaps":[{"begin":3,"length":1}]},{"pool":124,"snaps":[{"begin":3,"length":1},{"begin":5,"length":1}]},{"pool":125,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":126,"snaps":[{"begin":5,"length":2}]},{"pool":127,"snaps":[{"begin":4,"length":1}]},{"pool":128,"snaps":[{"begin":5,"length":2}]},{"pool":129,"snaps":[{"begin":4,"length":1}]},{"pool":130,"snaps":[{"begin":5,"length":2}]},{"pool":131,"snaps":[{"begin":5,"length":1}]},{"pool":132,"snaps":[{"begin":5,"length":2}]},{"pool":133,"snaps":[{"begin":2,"length":1}]},{"pool":134,"snaps":[{"begin":3,"length":1},{"begin":5,"length":1}]},{"pool":136,"snaps":[{"begin":3,"length":1}]},{"pool":137,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":138,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":139,"snaps":[{"begin":4,"length":1},{"begin":6,"length":1}]},{"pool":140,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":141,"snaps":[{"begin":4,"length":2}]},{"pool":142,"snaps":[{"begin":5,"length":3}]},{"pool":143,"snaps":[{"begin":4,"length":1},{"begin":6,"length":1}]},{"pool":144,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":145,"snaps":[{"begin":4,"length":1}]},{"pool":146,"snaps":[{"begin":5,"length":2}]},{"pool":149,"snaps":[{"begin":2,"length":1}]},{"pool":150,"snaps":[{"begin":3,"length":1}]},{"pool":151,"snaps":[{"begin":2,"length":1}]},{"pool":152,"snaps":[{"begin":3,"length":1}]},{"pool":154,"snaps":[{"begin":3,"length":1},{"begin":5,"length":1}]},{"pool":156,"snaps":[{"begin":3,"length":1}]},{"pool":158,"snaps":[{"begin":3,"length":1}]},{"pool":159,"snaps":[{"begin":2,"length":1}]},{"pool":160,"snaps":[{"begin":3,"length":1}]},{"pool":161,"snaps":[{"begin":2,"length":1}]},{"pool":162,"snaps":[{"begin":3,"length":1}]},{"pool":163,"snaps":[{"begin":2,"length":1}]},{"pool":164,"snaps":[{"begin":3,"length":1},{"begin":5,"length":1}]},{"pool":165,"snaps":[{"begin":4,"length":1}]},{"pool":166,"snaps":[{"begin":5,"length":2}]},{"pool":167,"snaps":[{"begin":4,"length":1}]},{"pool":168,"snaps":[{"begin":5,"length":2}]},{"pool":169,"snaps":[{"begin":4,"length":1}]},{"pool":170,"snaps":[{"begin":5,"length":2}]},{"pool":171,"snaps":[{"begin":5,"length":1}]},{"pool":172,"snaps":[{"begin":5,"length":2}]},{"pool":173,"snaps":[{"begin":2,"length":1}]},{"pool":174,"snaps":[{"begin":5,"length":1}]},{"pool":175,"snaps":[{"begin":4,"length":2}]},{"pool":176,"snaps":[{"begin":5,"length":2},{"begin":8,"length":1}]},{"pool":177,"snaps":[{"begin":4,"length":1}]},{"pool":178,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":179,"snaps":[{"begin":4,"length":1}]},{"pool":180,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":181,"snaps":[{"begin":4,"length":2}]},{"pool":182,"snaps":[{"begin":5,"length":3}]},{"pool":183,"snaps":[{"begin":4,"length":1}]},{"pool":184,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[{"pool":183,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}}
, 2021-10-27T19:33:03.159 INFO:tasks.ceph:Scrubbing osd.0
, 2021-10-27T19:33:03.160 DEBUG:teuthology.orchestra.run.gibba016:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 config set osd_debug_deep_scrub_sleep 0

Ignoring malparsed backtrace: 2021-10-27T19:33:04.435 INFO:tasks.ceph:pgid 147.1a last_scrub_stamp 2021-10-27T19:27:58.262979+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=27, tm_sec=58, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:33:04.436 INFO:tasks.ceph:pgid 148.1e last_scrub_stamp 2021-10-27T19:28:00.269882+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=28, tm_sec=0, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:33:04.436 INFO:tasks.ceph:pgid 147.19 last_scrub_stamp 2021-10-27T19:27:58.262979+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=27, tm_sec=58, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)

Ignoring malparsed backtrace: 2021-10-27T19:33:24.717 INFO:tasks.ceph:pgid 183.5 last_scrub_stamp 2021-10-27T19:32:40.331087+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=40, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:33:24.717 INFO:tasks.ceph:pgid 183.4 last_scrub_stamp 2021-10-27T19:32:40.331087+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=40, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:33:24.718 INFO:tasks.ceph:pgid 184.c last_scrub_stamp 2021-10-27T19:32:42.336433+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=42, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)

Ignoring malparsed backtrace: 2021-10-27T19:33:45.005 INFO:tasks.ceph:pgid 184.1f last_scrub_stamp 2021-10-27T19:32:42.336433+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=42, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:33:45.005 INFO:tasks.ceph:pgid 184.2 last_scrub_stamp 2021-10-27T19:32:42.336433+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=42, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:33:45.006 INFO:tasks.ceph:pgid 184.6 last_scrub_stamp 2021-10-27T19:32:42.336433+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=42, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)

Ignoring malparsed backtrace: 2021-10-27T19:34:05.285 INFO:tasks.ceph:pgid 183.8 last_scrub_stamp 2021-10-27T19:32:40.331087+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=40, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:34:05.285 INFO:tasks.ceph:pgid 184.c last_scrub_stamp 2021-10-27T19:32:42.336433+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=42, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:34:05.286 INFO:tasks.ceph:pgid 184.d last_scrub_stamp 2021-10-27T19:32:42.336433+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=42, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)

Ignoring malparsed backtrace: 2021-10-27T19:34:45.806 INFO:tasks.ceph:pgid 183.2 last_scrub_stamp 2021-10-27T19:32:40.331087+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=40, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:34:45.807 INFO:tasks.ceph:pgid 184.f last_scrub_stamp 2021-10-27T19:32:42.336433+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=42, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:34:45.807 INFO:tasks.ceph:pgid 148.18 last_scrub_stamp 2021-10-27T19:28:00.269882+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=28, tm_sec=0, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)

Ignoring malparsed backtrace:  ceph version 17.0.0-8454-ged9d0b44 (ed9d0b441738e5d5fee0b1394a629b504365ab69) quincy (dev)
,  1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x152) [0x5ff402c]
,  2: /usr/lib64/ceph/libceph-common.so.2(+0x28524d) [0x5ff424d]

Ignoring malparsed backtrace: ==30436==    by 0x4F00FB0: librados::v14_2_0::RadosClient::connect() (RadosClient.cc:312)
, ==30436==    by 0xFAA207: connect_cluster_pp(librados::v14_2_0::Rados&, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > const&) (test_cxx.cc:161)
, ==30436==    by 0xFAADFF: connect_cluster_pp[abi:cxx11](librados::v14_2_0::Rados&) (test_cxx.cc:126)

Ignoring malparsed backtrace: ==30436==    by 0x938CE24: ??? (in /usr/lib64/libnl-route-3.so.200.26.0)
, ==30436==    by 0x400F8B9: call_init.part.0 (in /usr/lib64/ld-2.28.so)
, ==30436==    by 0x400F9B9: _dl_init (in /usr/lib64/ld-2.28.so)

Ignoring malparsed backtrace: ==30436==    by 0xFCB676: Run (gtest.cc:2848)
, ==30436==    by 0xFCB676: testing::TestInfo::Run() (gtest.cc:2831)
, ==30436==    by 0xFCBC70: Run (gtest.cc:3012)

Ignoring malparsed backtrace: ==30436==    by 0x92C60E: librbd::ImageCtx::ImageCtx(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, char const*, librados::v14_2_0::IoCtx&, bool) (ImageCtx.cc:123)
, ==30436==    by 0x81F4BE: UnknownInlinedFun (ImageCtx.h:239)
, ==30436==    by 0x81F4BE: rbd::mirror::image_replayer::OpenImageRequest<librbd::ImageCtx>::send_open_image() (OpenImageRequest.cc:41)

Ignoring malparsed backtrace: ==30436==    by 0x6064F9A: KeyRing::set_modifier(char const*, char const*, EntityName&, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> > >&) (KeyRing.cc:106)
, ==30436==    by 0x60655DF: KeyRing::decode(ceph::buffer::v15_2_0::list::iterator_impl<true>&) (KeyRing.cc:193)
, ==30436==    by 0x6066474: KeyRing::load(ceph::common::CephContext*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) (KeyRing.cc:219)

Ignoring malparsed backtrace: ==30436==    by 0x6064BA2: encode (encoding.h:213)
, ==30436==    by 0x6064BA2: encode (encoding.h:219)
, ==30436==    by 0x6064BA2: KeyRing::set_modifier(char const*, char const*, EntityName&, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> > >&) (KeyRing.cc:117)

Ignoring malparsed backtrace: 2021-10-27T19:33:02.148 INFO:teuthology.orchestra.run.gibba016.stdout:{"epoch":1328,"fsid":"5a830528-8f3d-4eb2-add7-9b2182a4cd3c","created":"2021-10-27T19:09:35.021117+0000","modified":"2021-10-27T19:33:00.308113+0000","last_up_change":"2021-10-27T19:09:54.312852+0000","last_in_change":"2021-10-27T19:09:35.756621+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":184,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2021-10-27T19:09:54.736136+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2021-10-27T19:09:56.157326+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":147,"pool_name":"test-rbd-mirror-gibba016-30436-145","create_time":"2021-10-27T19:27:57.881233+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1029","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":148,"pool_name":"test-rbd-mirror-gibba016-30436-146","create_time":"2021-10-27T19:27:59.285331+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1031","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":183,"pool_name":"test-rados-api-gibba016-30436-181","create_time":"2021-10-27T19:32:39.980131+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1326","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":8,"snap_epoch":1326,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":184,"pool_name":"test-rados-api-gibba016-30436-182","create_time":"2021-10-27T19:32:42.015126+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1327","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":10,"snap_epoch":1327,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"a18d1ceb-edb8-41f1-803d-f4480b2941ed","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":1311,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6802","nonce":28946},{"type":"v1","addr":"172.21.2.116:6804","nonce":28946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6806","nonce":28946},{"type":"v1","addr":"172.21.2.116:6808","nonce":28946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6814","nonce":28946},{"type":"v1","addr":"172.21.2.116:6815","nonce":28946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6810","nonce":28946},{"type":"v1","addr":"172.21.2.116:6813","nonce":28946}]},"public_addr":"172.21.2.116:6804/28946","cluster_addr":"172.21.2.116:6808/28946","heartbeat_back_addr":"172.21.2.116:6815/28946","heartbeat_front_addr":"172.21.2.116:6813/28946","state":["exists","up"]},{"osd":1,"uuid":"7545bf9e-4f37-4aa5-b2dc-d5994995e31c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":1311,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6800","nonce":28947},{"type":"v1","addr":"172.21.2.116:6801","nonce":28947}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6803","nonce":28947},{"type":"v1","addr":"172.21.2.116:6805","nonce":28947}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6811","nonce":28947},{"type":"v1","addr":"172.21.2.116:6812","nonce":28947}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6807","nonce":28947},{"type":"v1","addr":"172.21.2.116:6809","nonce":28947}]},"public_addr":"172.21.2.116:6801/28947","cluster_addr":"172.21.2.116:6805/28947","heartbeat_back_addr":"172.21.2.116:6812/28947","heartbeat_front_addr":"172.21.2.116:6809/28947","state":["exists","up"]},{"osd":2,"uuid":"c1f6b3b1-c86b-449f-aacf-d757769f5caf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":1311,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6816","nonce":28948},{"type":"v1","addr":"172.21.2.116:6817","nonce":28948}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6818","nonce":28948},{"type":"v1","addr":"172.21.2.116:6819","nonce":28948}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6822","nonce":28948},{"type":"v1","addr":"172.21.2.116:6823","nonce":28948}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6820","nonce":28948},{"type":"v1","addr":"172.21.2.116:6821","nonce":28948}]},"public_addr":"172.21.2.116:6817/28948","cluster_addr":"172.21.2.116:6819/28948","heartbeat_back_addr":"172.21.2.116:6823/28948","heartbeat_front_addr":"172.21.2.116:6821/28948","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-10-27T19:09:36.834954+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-10-27T19:09:36.848998+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-10-27T19:09:36.848327+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"147.5","mappings":[{"from":2,"to":0}]},{"pgid":"147.1b","mappings":[{"from":0,"to":1}]},{"pgid":"147.1e","mappings":[{"from":2,"to":1}]}],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":24,"snaps":[{"begin":2,"length":1}]},{"pool":25,"snaps":[{"begin":4,"length":2},{"begin":8,"length":2},{"begin":12,"length":1}]},{"pool":29,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":30,"snaps":[{"begin":3,"length":1},{"begin":5,"length":1}]},{"pool":31,"snaps":[{"begin":2,"length":1}]},{"pool":32,"snaps":[{"begin":3,"length":1}]},{"pool":34,"snaps":[{"begin":3,"length":1}]},{"pool":36,"snaps":[{"begin":3,"length":1}]},{"pool":38,"snaps":[{"begin":3,"length":1}]},{"pool":39,"snaps":[{"begin":2,"length":1}]},{"pool":40,"snaps":[{"begin":3,"length":1}]},{"pool":41,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":42,"snaps":[{"begin":5,"length":1}]},{"pool":43,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":44,"snaps":[{"begin":5,"length":1}]},{"pool":45,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":46,"snaps":[{"begin":5,"length":1}]},{"pool":47,"snaps":[{"begin":4,"length":1},{"begin":6,"length":1}]},{"pool":48,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":49,"snaps":[{"begin":4,"length":1},{"begin":6,"length":1}]},{"pool":50,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":51,"snaps":[{"begin":6,"length":1}]},{"pool":52,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":53,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":54,"snaps":[{"begin":5,"length":1}]},{"pool":55,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":56,"snaps":[{"begin":5,"length":1}]},{"pool":57,"snaps":[{"begin":5,"length":1}]},{"pool":58,"snaps":[{"begin":6,"length":1}]},{"pool":59,"snaps":[{"begin":2,"length":1},{"begin":5,"length":1}]},{"pool":60,"snaps":[{"begin":6,"length":1}]},{"pool":61,"snaps":[{"begin":4,"length":2}]},{"pool":62,"snaps":[{"begin":5,"length":2}]},{"pool":63,"snaps":[{"begin":5,"length":1}]},{"pool":64,"snaps":[{"begin":6,"length":1}]},{"pool":65,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":66,"snaps":[{"begin":5,"length":1}]},{"pool":69,"snaps":[{"begin":2,"length":1}]},{"pool":70,"snaps":[{"begin":3,"length":1}]},{"pool":71,"snaps":[{"begin":2,"length":1}]},{"pool":72,"snaps":[{"begin":3,"length":1}]},{"pool":74,"snaps":[{"begin":3,"length":1},{"begin":5,"length":1}]},{"pool":76,"snaps":[{"begin":3,"length":1}]},{"pool":78,"snaps":[{"begin":3,"length":1}]},{"pool":79,"snaps":[{"begin":2,"length":1}]},{"pool":80,"snaps":[{"begin":3,"length":1}]},{"pool":81,"snaps":[{"begin":2,"length":1}]},{"pool":82,"snaps":[{"begin":3,"length":1}]},{"pool":83,"snaps":[{"begin":2,"length":1}]},{"pool":84,"snaps":[{"begin":5,"length":1}]},{"pool":85,"snaps":[{"begin":4,"length":1}]},{"pool":86,"snaps":[{"begin":5,"length":2}]},{"pool":87,"snaps":[{"begin":4,"length":1}]},{"pool":88,"snaps":[{"begin":5,"length":2}]},{"pool":89,"snaps":[{"begin":4,"length":1}]},{"pool":90,"snaps":[{"begin":5,"length":2}]},{"pool":91,"snaps":[{"begin":5,"length":2}]},{"pool":92,"snaps":[{"begin":5,"length":2}]},{"pool":93,"snaps":[{"begin":2,"length":1}]},{"pool":94,"snaps":[{"begin":5,"length":1}]},{"pool":96,"snaps":[{"begin":3,"length":1}]},{"pool":97,"snaps":[{"begin":4,"length":1}]},{"pool":98,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":99,"snaps":[{"begin":4,"length":1}]},{"pool":100,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":101,"snaps":[{"begin":4,"length":2}]},{"pool":102,"snaps":[{"begin":5,"length":3}]},{"pool":103,"snaps":[{"begin":4,"length":1},{"begin":6,"length":1}]},{"pool":104,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":105,"snaps":[{"begin":4,"length":1}]},{"pool":106,"snaps":[{"begin":5,"length":2}]},{"pool":109,"snaps":[{"begin":2,"length":1}]},{"pool":110,"snaps":[{"begin":3,"length":1}]},{"pool":111,"snaps":[{"begin":2,"length":1}]},{"pool":112,"snaps":[{"begin":3,"length":1}]},{"pool":114,"snaps":[{"begin":3,"length":1},{"begin":5,"length":1}]},{"pool":116,"snaps":[{"begin":3,"length":1}]},{"pool":118,"snaps":[{"begin":3,"length":1}]},{"pool":119,"snaps":[{"begin":2,"length":1}]},{"pool":120,"snaps":[{"begin":3,"length":1}]},{"pool":121,"snaps":[{"begin":2,"length":1}]},{"pool":122,"snaps":[{"begin":3,"length":1}]},{"pool":124,"snaps":[{"begin":3,"length":1},{"begin":5,"length":1}]},{"pool":125,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":126,"snaps":[{"begin":5,"length":2}]},{"pool":127,"snaps":[{"begin":4,"length":1}]},{"pool":128,"snaps":[{"begin":5,"length":2}]},{"pool":129,"snaps":[{"begin":4,"length":1}]},{"pool":130,"snaps":[{"begin":5,"length":2}]},{"pool":131,"snaps":[{"begin":5,"length":1}]},{"pool":132,"snaps":[{"begin":5,"length":2}]},{"pool":133,"snaps":[{"begin":2,"length":1}]},{"pool":134,"snaps":[{"begin":3,"length":1},{"begin":5,"length":1}]},{"pool":136,"snaps":[{"begin":3,"length":1}]},{"pool":137,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":138,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":139,"snaps":[{"begin":4,"length":1},{"begin":6,"length":1}]},{"pool":140,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":141,"snaps":[{"begin":4,"length":2}]},{"pool":142,"snaps":[{"begin":5,"length":3}]},{"pool":143,"snaps":[{"begin":4,"length":1},{"begin":6,"length":1}]},{"pool":144,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":145,"snaps":[{"begin":4,"length":1}]},{"pool":146,"snaps":[{"begin":5,"length":2}]},{"pool":149,"snaps":[{"begin":2,"length":1}]},{"pool":150,"snaps":[{"begin":3,"length":1}]},{"pool":151,"snaps":[{"begin":2,"length":1}]},{"pool":152,"snaps":[{"begin":3,"length":1}]},{"pool":154,"snaps":[{"begin":3,"length":1},{"begin":5,"length":1}]},{"pool":156,"snaps":[{"begin":3,"length":1}]},{"pool":158,"snaps":[{"begin":3,"length":1}]},{"pool":159,"snaps":[{"begin":2,"length":1}]},{"pool":160,"snaps":[{"begin":3,"length":1}]},{"pool":161,"snaps":[{"begin":2,"length":1}]},{"pool":162,"snaps":[{"begin":3,"length":1}]},{"pool":163,"snaps":[{"begin":2,"length":1}]},{"pool":164,"snaps":[{"begin":3,"length":1},{"begin":5,"length":1}]},{"pool":165,"snaps":[{"begin":4,"length":1}]},{"pool":166,"snaps":[{"begin":5,"length":2}]},{"pool":167,"snaps":[{"begin":4,"length":1}]},{"pool":168,"snaps":[{"begin":5,"length":2}]},{"pool":169,"snaps":[{"begin":4,"length":1}]},{"pool":170,"snaps":[{"begin":5,"length":2}]},{"pool":171,"snaps":[{"begin":5,"length":1}]},{"pool":172,"snaps":[{"begin":5,"length":2}]},{"pool":173,"snaps":[{"begin":2,"length":1}]},{"pool":174,"snaps":[{"begin":5,"length":1}]},{"pool":175,"snaps":[{"begin":4,"length":2}]},{"pool":176,"snaps":[{"begin":5,"length":2},{"begin":8,"length":1}]},{"pool":177,"snaps":[{"begin":4,"length":1}]},{"pool":178,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":179,"snaps":[{"begin":4,"length":1}]},{"pool":180,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":181,"snaps":[{"begin":4,"length":2}]},{"pool":182,"snaps":[{"begin":5,"length":3}]},{"pool":183,"snaps":[{"begin":4,"length":1}]},{"pool":184,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[{"pool":183,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}}
, 2021-10-27T19:33:03.159 INFO:tasks.ceph:Scrubbing osd.0
, 2021-10-27T19:33:03.160 DEBUG:teuthology.orchestra.run.gibba016:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 config set osd_debug_deep_scrub_sleep 0

Ignoring malparsed backtrace: 2021-10-27T19:33:04.435 INFO:tasks.ceph:pgid 147.1a last_scrub_stamp 2021-10-27T19:27:58.262979+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=27, tm_sec=58, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:33:04.436 INFO:tasks.ceph:pgid 148.1e last_scrub_stamp 2021-10-27T19:28:00.269882+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=28, tm_sec=0, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:33:04.436 INFO:tasks.ceph:pgid 147.19 last_scrub_stamp 2021-10-27T19:27:58.262979+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=27, tm_sec=58, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)

Ignoring malparsed backtrace: 2021-10-27T19:33:24.717 INFO:tasks.ceph:pgid 183.5 last_scrub_stamp 2021-10-27T19:32:40.331087+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=40, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:33:24.717 INFO:tasks.ceph:pgid 183.4 last_scrub_stamp 2021-10-27T19:32:40.331087+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=40, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:33:24.718 INFO:tasks.ceph:pgid 184.c last_scrub_stamp 2021-10-27T19:32:42.336433+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=42, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)

Ignoring malparsed backtrace: 2021-10-27T19:33:45.005 INFO:tasks.ceph:pgid 184.1f last_scrub_stamp 2021-10-27T19:32:42.336433+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=42, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:33:45.005 INFO:tasks.ceph:pgid 184.2 last_scrub_stamp 2021-10-27T19:32:42.336433+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=42, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:33:45.006 INFO:tasks.ceph:pgid 184.6 last_scrub_stamp 2021-10-27T19:32:42.336433+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=42, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)

Ignoring malparsed backtrace: 2021-10-27T19:34:05.285 INFO:tasks.ceph:pgid 183.8 last_scrub_stamp 2021-10-27T19:32:40.331087+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=40, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:34:05.285 INFO:tasks.ceph:pgid 184.c last_scrub_stamp 2021-10-27T19:32:42.336433+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=42, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:34:05.286 INFO:tasks.ceph:pgid 184.d last_scrub_stamp 2021-10-27T19:32:42.336433+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=42, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)

Ignoring malparsed backtrace: 2021-10-27T19:34:45.806 INFO:tasks.ceph:pgid 183.2 last_scrub_stamp 2021-10-27T19:32:40.331087+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=40, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:34:45.807 INFO:tasks.ceph:pgid 184.f last_scrub_stamp 2021-10-27T19:32:42.336433+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=42, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:34:45.807 INFO:tasks.ceph:pgid 148.18 last_scrub_stamp 2021-10-27T19:28:00.269882+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=28, tm_sec=0, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)

Ignoring malparsed backtrace:  ceph version 17.0.0-8454-ged9d0b44 (ed9d0b441738e5d5fee0b1394a629b504365ab69) quincy (dev)
,  1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x152) [0x5ff402c]
,  2: /usr/lib64/ceph/libceph-common.so.2(+0x28524d) [0x5ff424d]

Ignoring malparsed backtrace: ==30436==    by 0x4F00FB0: librados::v14_2_0::RadosClient::connect() (RadosClient.cc:312)
, ==30436==    by 0xFAA207: connect_cluster_pp(librados::v14_2_0::Rados&, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > const&) (test_cxx.cc:161)
, ==30436==    by 0xFAADFF: connect_cluster_pp[abi:cxx11](librados::v14_2_0::Rados&) (test_cxx.cc:126)

Ignoring malparsed backtrace: ==30436==    by 0x938CE24: ??? (in /usr/lib64/libnl-route-3.so.200.26.0)
, ==30436==    by 0x400F8B9: call_init.part.0 (in /usr/lib64/ld-2.28.so)
, ==30436==    by 0x400F9B9: _dl_init (in /usr/lib64/ld-2.28.so)

Ignoring malparsed backtrace: ==30436==    by 0xFCB676: Run (gtest.cc:2848)
, ==30436==    by 0xFCB676: testing::TestInfo::Run() (gtest.cc:2831)
, ==30436==    by 0xFCBC70: Run (gtest.cc:3012)

Ignoring malparsed backtrace: ==30436==    by 0x92C60E: librbd::ImageCtx::ImageCtx(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, char const*, librados::v14_2_0::IoCtx&, bool) (ImageCtx.cc:123)
, ==30436==    by 0x81F4BE: UnknownInlinedFun (ImageCtx.h:239)
, ==30436==    by 0x81F4BE: rbd::mirror::image_replayer::OpenImageRequest<librbd::ImageCtx>::send_open_image() (OpenImageRequest.cc:41)

Ignoring malparsed backtrace: ==30436==    by 0x6064F9A: KeyRing::set_modifier(char const*, char const*, EntityName&, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> > >&) (KeyRing.cc:106)
, ==30436==    by 0x60655DF: KeyRing::decode(ceph::buffer::v15_2_0::list::iterator_impl<true>&) (KeyRing.cc:193)
, ==30436==    by 0x6066474: KeyRing::load(ceph::common::CephContext*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) (KeyRing.cc:219)

Ignoring malparsed backtrace: ==30436==    by 0x6064BA2: encode (encoding.h:213)
, ==30436==    by 0x6064BA2: encode (encoding.h:219)
, ==30436==    by 0x6064BA2: KeyRing::set_modifier(char const*, char const*, EntityName&, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> > >&) (KeyRing.cc:117)

Ignoring malparsed backtrace: 2021-10-27T19:33:02.148 INFO:teuthology.orchestra.run.gibba016.stdout:{"epoch":1328,"fsid":"5a830528-8f3d-4eb2-add7-9b2182a4cd3c","created":"2021-10-27T19:09:35.021117+0000","modified":"2021-10-27T19:33:00.308113+0000","last_up_change":"2021-10-27T19:09:54.312852+0000","last_in_change":"2021-10-27T19:09:35.756621+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":184,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2021-10-27T19:09:54.736136+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2021-10-27T19:09:56.157326+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":147,"pool_name":"test-rbd-mirror-gibba016-30436-145","create_time":"2021-10-27T19:27:57.881233+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1029","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":148,"pool_name":"test-rbd-mirror-gibba016-30436-146","create_time":"2021-10-27T19:27:59.285331+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1031","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":183,"pool_name":"test-rados-api-gibba016-30436-181","create_time":"2021-10-27T19:32:39.980131+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1326","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":8,"snap_epoch":1326,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":184,"pool_name":"test-rados-api-gibba016-30436-182","create_time":"2021-10-27T19:32:42.015126+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1327","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":10,"snap_epoch":1327,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"a18d1ceb-edb8-41f1-803d-f4480b2941ed","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":1311,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6802","nonce":28946},{"type":"v1","addr":"172.21.2.116:6804","nonce":28946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6806","nonce":28946},{"type":"v1","addr":"172.21.2.116:6808","nonce":28946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6814","nonce":28946},{"type":"v1","addr":"172.21.2.116:6815","nonce":28946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6810","nonce":28946},{"type":"v1","addr":"172.21.2.116:6813","nonce":28946}]},"public_addr":"172.21.2.116:6804/28946","cluster_addr":"172.21.2.116:6808/28946","heartbeat_back_addr":"172.21.2.116:6815/28946","heartbeat_front_addr":"172.21.2.116:6813/28946","state":["exists","up"]},{"osd":1,"uuid":"7545bf9e-4f37-4aa5-b2dc-d5994995e31c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":1311,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6800","nonce":28947},{"type":"v1","addr":"172.21.2.116:6801","nonce":28947}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6803","nonce":28947},{"type":"v1","addr":"172.21.2.116:6805","nonce":28947}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6811","nonce":28947},{"type":"v1","addr":"172.21.2.116:6812","nonce":28947}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6807","nonce":28947},{"type":"v1","addr":"172.21.2.116:6809","nonce":28947}]},"public_addr":"172.21.2.116:6801/28947","cluster_addr":"172.21.2.116:6805/28947","heartbeat_back_addr":"172.21.2.116:6812/28947","heartbeat_front_addr":"172.21.2.116:6809/28947","state":["exists","up"]},{"osd":2,"uuid":"c1f6b3b1-c86b-449f-aacf-d757769f5caf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":1311,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6816","nonce":28948},{"type":"v1","addr":"172.21.2.116:6817","nonce":28948}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6818","nonce":28948},{"type":"v1","addr":"172.21.2.116:6819","nonce":28948}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6822","nonce":28948},{"type":"v1","addr":"172.21.2.116:6823","nonce":28948}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6820","nonce":28948},{"type":"v1","addr":"172.21.2.116:6821","nonce":28948}]},"public_addr":"172.21.2.116:6817/28948","cluster_addr":"172.21.2.116:6819/28948","heartbeat_back_addr":"172.21.2.116:6823/28948","heartbeat_front_addr":"172.21.2.116:6821/28948","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-10-27T19:09:36.834954+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-10-27T19:09:36.848998+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-10-27T19:09:36.848327+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"147.5","mappings":[{"from":2,"to":0}]},{"pgid":"147.1b","mappings":[{"from":0,"to":1}]},{"pgid":"147.1e","mappings":[{"from":2,"to":1}]}],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":24,"snaps":[{"begin":2,"length":1}]},{"pool":25,"snaps":[{"begin":4,"length":2},{"begin":8,"length":2},{"begin":12,"length":1}]},{"pool":29,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":30,"snaps":[{"begin":3,"length":1},{"begin":5,"length":1}]},{"pool":31,"snaps":[{"begin":2,"length":1}]},{"pool":32,"snaps":[{"begin":3,"length":1}]},{"pool":34,"snaps":[{"begin":3,"length":1}]},{"pool":36,"snaps":[{"begin":3,"length":1}]},{"pool":38,"snaps":[{"begin":3,"length":1}]},{"pool":39,"snaps":[{"begin":2,"length":1}]},{"pool":40,"snaps":[{"begin":3,"length":1}]},{"pool":41,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":42,"snaps":[{"begin":5,"length":1}]},{"pool":43,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":44,"snaps":[{"begin":5,"length":1}]},{"pool":45,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":46,"snaps":[{"begin":5,"length":1}]},{"pool":47,"snaps":[{"begin":4,"length":1},{"begin":6,"length":1}]},{"pool":48,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":49,"snaps":[{"begin":4,"length":1},{"begin":6,"length":1}]},{"pool":50,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":51,"snaps":[{"begin":6,"length":1}]},{"pool":52,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":53,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":54,"snaps":[{"begin":5,"length":1}]},{"pool":55,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":56,"snaps":[{"begin":5,"length":1}]},{"pool":57,"snaps":[{"begin":5,"length":1}]},{"pool":58,"snaps":[{"begin":6,"length":1}]},{"pool":59,"snaps":[{"begin":2,"length":1},{"begin":5,"length":1}]},{"pool":60,"snaps":[{"begin":6,"length":1}]},{"pool":61,"snaps":[{"begin":4,"length":2}]},{"pool":62,"snaps":[{"begin":5,"length":2}]},{"pool":63,"snaps":[{"begin":5,"length":1}]},{"pool":64,"snaps":[{"begin":6,"length":1}]},{"pool":65,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":66,"snaps":[{"begin":5,"length":1}]},{"pool":69,"snaps":[{"begin":2,"length":1}]},{"pool":70,"snaps":[{"begin":3,"length":1}]},{"pool":71,"snaps":[{"begin":2,"length":1}]},{"pool":72,"snaps":[{"begin":3,"length":1}]},{"pool":74,"snaps":[{"begin":3,"length":1},{"begin":5,"length":1}]},{"pool":76,"snaps":[{"begin":3,"length":1}]},{"pool":78,"snaps":[{"begin":3,"length":1}]},{"pool":79,"snaps":[{"begin":2,"length":1}]},{"pool":80,"snaps":[{"begin":3,"length":1}]},{"pool":81,"snaps":[{"begin":2,"length":1}]},{"pool":82,"snaps":[{"begin":3,"length":1}]},{"pool":83,"snaps":[{"begin":2,"length":1}]},{"pool":84,"snaps":[{"begin":5,"length":1}]},{"pool":85,"snaps":[{"begin":4,"length":1}]},{"pool":86,"snaps":[{"begin":5,"length":2}]},{"pool":87,"snaps":[{"begin":4,"length":1}]},{"pool":88,"snaps":[{"begin":5,"length":2}]},{"pool":89,"snaps":[{"begin":4,"length":1}]},{"pool":90,"snaps":[{"begin":5,"length":2}]},{"pool":91,"snaps":[{"begin":5,"length":2}]},{"pool":92,"snaps":[{"begin":5,"length":2}]},{"pool":93,"snaps":[{"begin":2,"length":1}]},{"pool":94,"snaps":[{"begin":5,"length":1}]},{"pool":96,"snaps":[{"begin":3,"length":1}]},{"pool":97,"snaps":[{"begin":4,"length":1}]},{"pool":98,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":99,"snaps":[{"begin":4,"length":1}]},{"pool":100,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":101,"snaps":[{"begin":4,"length":2}]},{"pool":102,"snaps":[{"begin":5,"length":3}]},{"pool":103,"snaps":[{"begin":4,"length":1},{"begin":6,"length":1}]},{"pool":104,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":105,"snaps":[{"begin":4,"length":1}]},{"pool":106,"snaps":[{"begin":5,"length":2}]},{"pool":109,"snaps":[{"begin":2,"length":1}]},{"pool":110,"snaps":[{"begin":3,"length":1}]},{"pool":111,"snaps":[{"begin":2,"length":1}]},{"pool":112,"snaps":[{"begin":3,"length":1}]},{"pool":114,"snaps":[{"begin":3,"length":1},{"begin":5,"length":1}]},{"pool":116,"snaps":[{"begin":3,"length":1}]},{"pool":118,"snaps":[{"begin":3,"length":1}]},{"pool":119,"snaps":[{"begin":2,"length":1}]},{"pool":120,"snaps":[{"begin":3,"length":1}]},{"pool":121,"snaps":[{"begin":2,"length":1}]},{"pool":122,"snaps":[{"begin":3,"length":1}]},{"pool":124,"snaps":[{"begin":3,"length":1},{"begin":5,"length":1}]},{"pool":125,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":126,"snaps":[{"begin":5,"length":2}]},{"pool":127,"snaps":[{"begin":4,"length":1}]},{"pool":128,"snaps":[{"begin":5,"length":2}]},{"pool":129,"snaps":[{"begin":4,"length":1}]},{"pool":130,"snaps":[{"begin":5,"length":2}]},{"pool":131,"snaps":[{"begin":5,"length":1}]},{"pool":132,"snaps":[{"begin":5,"length":2}]},{"pool":133,"snaps":[{"begin":2,"length":1}]},{"pool":134,"snaps":[{"begin":3,"length":1},{"begin":5,"length":1}]},{"pool":136,"snaps":[{"begin":3,"length":1}]},{"pool":137,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":138,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":139,"snaps":[{"begin":4,"length":1},{"begin":6,"length":1}]},{"pool":140,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":141,"snaps":[{"begin":4,"length":2}]},{"pool":142,"snaps":[{"begin":5,"length":3}]},{"pool":143,"snaps":[{"begin":4,"length":1},{"begin":6,"length":1}]},{"pool":144,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":145,"snaps":[{"begin":4,"length":1}]},{"pool":146,"snaps":[{"begin":5,"length":2}]},{"pool":149,"snaps":[{"begin":2,"length":1}]},{"pool":150,"snaps":[{"begin":3,"length":1}]},{"pool":151,"snaps":[{"begin":2,"length":1}]},{"pool":152,"snaps":[{"begin":3,"length":1}]},{"pool":154,"snaps":[{"begin":3,"length":1},{"begin":5,"length":1}]},{"pool":156,"snaps":[{"begin":3,"length":1}]},{"pool":158,"snaps":[{"begin":3,"length":1}]},{"pool":159,"snaps":[{"begin":2,"length":1}]},{"pool":160,"snaps":[{"begin":3,"length":1}]},{"pool":161,"snaps":[{"begin":2,"length":1}]},{"pool":162,"snaps":[{"begin":3,"length":1}]},{"pool":163,"snaps":[{"begin":2,"length":1}]},{"pool":164,"snaps":[{"begin":3,"length":1},{"begin":5,"length":1}]},{"pool":165,"snaps":[{"begin":4,"length":1}]},{"pool":166,"snaps":[{"begin":5,"length":2}]},{"pool":167,"snaps":[{"begin":4,"length":1}]},{"pool":168,"snaps":[{"begin":5,"length":2}]},{"pool":169,"snaps":[{"begin":4,"length":1}]},{"pool":170,"snaps":[{"begin":5,"length":2}]},{"pool":171,"snaps":[{"begin":5,"length":1}]},{"pool":172,"snaps":[{"begin":5,"length":2}]},{"pool":173,"snaps":[{"begin":2,"length":1}]},{"pool":174,"snaps":[{"begin":5,"length":1}]},{"pool":175,"snaps":[{"begin":4,"length":2}]},{"pool":176,"snaps":[{"begin":5,"length":2},{"begin":8,"length":1}]},{"pool":177,"snaps":[{"begin":4,"length":1}]},{"pool":178,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":179,"snaps":[{"begin":4,"length":1}]},{"pool":180,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":181,"snaps":[{"begin":4,"length":2}]},{"pool":182,"snaps":[{"begin":5,"length":3}]},{"pool":183,"snaps":[{"begin":4,"length":1}]},{"pool":184,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[{"pool":183,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}}
, 2021-10-27T19:33:03.159 INFO:tasks.ceph:Scrubbing osd.0
, 2021-10-27T19:33:03.160 DEBUG:teuthology.orchestra.run.gibba016:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 config set osd_debug_deep_scrub_sleep 0

Ignoring malparsed backtrace: 2021-10-27T19:33:04.435 INFO:tasks.ceph:pgid 147.1a last_scrub_stamp 2021-10-27T19:27:58.262979+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=27, tm_sec=58, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:33:04.436 INFO:tasks.ceph:pgid 148.1e last_scrub_stamp 2021-10-27T19:28:00.269882+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=28, tm_sec=0, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:33:04.436 INFO:tasks.ceph:pgid 147.19 last_scrub_stamp 2021-10-27T19:27:58.262979+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=27, tm_sec=58, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)

Ignoring malparsed backtrace: 2021-10-27T19:33:24.717 INFO:tasks.ceph:pgid 183.5 last_scrub_stamp 2021-10-27T19:32:40.331087+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=40, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:33:24.717 INFO:tasks.ceph:pgid 183.4 last_scrub_stamp 2021-10-27T19:32:40.331087+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=40, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:33:24.718 INFO:tasks.ceph:pgid 184.c last_scrub_stamp 2021-10-27T19:32:42.336433+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=42, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)

Ignoring malparsed backtrace: 2021-10-27T19:33:45.005 INFO:tasks.ceph:pgid 184.1f last_scrub_stamp 2021-10-27T19:32:42.336433+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=42, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:33:45.005 INFO:tasks.ceph:pgid 184.2 last_scrub_stamp 2021-10-27T19:32:42.336433+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=42, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:33:45.006 INFO:tasks.ceph:pgid 184.6 last_scrub_stamp 2021-10-27T19:32:42.336433+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=42, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)

Ignoring malparsed backtrace: 2021-10-27T19:34:05.285 INFO:tasks.ceph:pgid 183.8 last_scrub_stamp 2021-10-27T19:32:40.331087+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=40, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:34:05.285 INFO:tasks.ceph:pgid 184.c last_scrub_stamp 2021-10-27T19:32:42.336433+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=42, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:34:05.286 INFO:tasks.ceph:pgid 184.d last_scrub_stamp 2021-10-27T19:32:42.336433+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=42, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)

Ignoring malparsed backtrace: 2021-10-27T19:34:45.806 INFO:tasks.ceph:pgid 183.2 last_scrub_stamp 2021-10-27T19:32:40.331087+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=40, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:34:45.807 INFO:tasks.ceph:pgid 184.f last_scrub_stamp 2021-10-27T19:32:42.336433+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=42, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:34:45.807 INFO:tasks.ceph:pgid 148.18 last_scrub_stamp 2021-10-27T19:28:00.269882+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=28, tm_sec=0, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)

Ignoring malparsed backtrace:  ceph version 17.0.0-8454-ged9d0b44 (ed9d0b441738e5d5fee0b1394a629b504365ab69) quincy (dev)
,  1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x152) [0x5ff402c]
,  2: /usr/lib64/ceph/libceph-common.so.2(+0x28524d) [0x5ff424d]

Ignoring malparsed backtrace: ==30436==    by 0x4F00FB0: librados::v14_2_0::RadosClient::connect() (RadosClient.cc:312)
, ==30436==    by 0xFAA207: connect_cluster_pp(librados::v14_2_0::Rados&, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > const&) (test_cxx.cc:161)
, ==30436==    by 0xFAADFF: connect_cluster_pp[abi:cxx11](librados::v14_2_0::Rados&) (test_cxx.cc:126)

Ignoring malparsed backtrace: ==30436==    by 0x938CE24: ??? (in /usr/lib64/libnl-route-3.so.200.26.0)
, ==30436==    by 0x400F8B9: call_init.part.0 (in /usr/lib64/ld-2.28.so)
, ==30436==    by 0x400F9B9: _dl_init (in /usr/lib64/ld-2.28.so)

Ignoring malparsed backtrace: ==30436==    by 0xFCB676: Run (gtest.cc:2848)
, ==30436==    by 0xFCB676: testing::TestInfo::Run() (gtest.cc:2831)
, ==30436==    by 0xFCBC70: Run (gtest.cc:3012)

Ignoring malparsed backtrace: ==30436==    by 0x92C60E: librbd::ImageCtx::ImageCtx(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, char const*, librados::v14_2_0::IoCtx&, bool) (ImageCtx.cc:123)
, ==30436==    by 0x81F4BE: UnknownInlinedFun (ImageCtx.h:239)
, ==30436==    by 0x81F4BE: rbd::mirror::image_replayer::OpenImageRequest<librbd::ImageCtx>::send_open_image() (OpenImageRequest.cc:41)

Ignoring malparsed backtrace: ==30436==    by 0x6064F9A: KeyRing::set_modifier(char const*, char const*, EntityName&, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> > >&) (KeyRing.cc:106)
, ==30436==    by 0x60655DF: KeyRing::decode(ceph::buffer::v15_2_0::list::iterator_impl<true>&) (KeyRing.cc:193)
, ==30436==    by 0x6066474: KeyRing::load(ceph::common::CephContext*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) (KeyRing.cc:219)

Ignoring malparsed backtrace: ==30436==    by 0x6064BA2: encode (encoding.h:213)
, ==30436==    by 0x6064BA2: encode (encoding.h:219)
, ==30436==    by 0x6064BA2: KeyRing::set_modifier(char const*, char const*, EntityName&, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> > >&) (KeyRing.cc:117)

Ignoring malparsed backtrace: 2021-10-27T19:33:02.148 INFO:teuthology.orchestra.run.gibba016.stdout:{"epoch":1328,"fsid":"5a830528-8f3d-4eb2-add7-9b2182a4cd3c","created":"2021-10-27T19:09:35.021117+0000","modified":"2021-10-27T19:33:00.308113+0000","last_up_change":"2021-10-27T19:09:54.312852+0000","last_in_change":"2021-10-27T19:09:35.756621+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":184,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2021-10-27T19:09:54.736136+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2021-10-27T19:09:56.157326+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":147,"pool_name":"test-rbd-mirror-gibba016-30436-145","create_time":"2021-10-27T19:27:57.881233+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1029","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":148,"pool_name":"test-rbd-mirror-gibba016-30436-146","create_time":"2021-10-27T19:27:59.285331+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1031","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":183,"pool_name":"test-rados-api-gibba016-30436-181","create_time":"2021-10-27T19:32:39.980131+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1326","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":8,"snap_epoch":1326,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":184,"pool_name":"test-rados-api-gibba016-30436-182","create_time":"2021-10-27T19:32:42.015126+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1327","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":10,"snap_epoch":1327,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"a18d1ceb-edb8-41f1-803d-f4480b2941ed","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":1311,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6802","nonce":28946},{"type":"v1","addr":"172.21.2.116:6804","nonce":28946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6806","nonce":28946},{"type":"v1","addr":"172.21.2.116:6808","nonce":28946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6814","nonce":28946},{"type":"v1","addr":"172.21.2.116:6815","nonce":28946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6810","nonce":28946},{"type":"v1","addr":"172.21.2.116:6813","nonce":28946}]},"public_addr":"172.21.2.116:6804/28946","cluster_addr":"172.21.2.116:6808/28946","heartbeat_back_addr":"172.21.2.116:6815/28946","heartbeat_front_addr":"172.21.2.116:6813/28946","state":["exists","up"]},{"osd":1,"uuid":"7545bf9e-4f37-4aa5-b2dc-d5994995e31c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":1311,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6800","nonce":28947},{"type":"v1","addr":"172.21.2.116:6801","nonce":28947}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6803","nonce":28947},{"type":"v1","addr":"172.21.2.116:6805","nonce":28947}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6811","nonce":28947},{"type":"v1","addr":"172.21.2.116:6812","nonce":28947}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6807","nonce":28947},{"type":"v1","addr":"172.21.2.116:6809","nonce":28947}]},"public_addr":"172.21.2.116:6801/28947","cluster_addr":"172.21.2.116:6805/28947","heartbeat_back_addr":"172.21.2.116:6812/28947","heartbeat_front_addr":"172.21.2.116:6809/28947","state":["exists","up"]},{"osd":2,"uuid":"c1f6b3b1-c86b-449f-aacf-d757769f5caf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":1311,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6816","nonce":28948},{"type":"v1","addr":"172.21.2.116:6817","nonce":28948}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6818","nonce":28948},{"type":"v1","addr":"172.21.2.116:6819","nonce":28948}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6822","nonce":28948},{"type":"v1","addr":"172.21.2.116:6823","nonce":28948}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6820","nonce":28948},{"type":"v1","addr":"172.21.2.116:6821","nonce":28948}]},"public_addr":"172.21.2.116:6817/28948","cluster_addr":"172.21.2.116:6819/28948","heartbeat_back_addr":"172.21.2.116:6823/28948","heartbeat_front_addr":"172.21.2.116:6821/28948","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-10-27T19:09:36.834954+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-10-27T19:09:36.848998+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-10-27T19:09:36.848327+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"147.5","mappings":[{"from":2,"to":0}]},{"pgid":"147.1b","mappings":[{"from":0,"to":1}]},{"pgid":"147.1e","mappings":[{"from":2,"to":1}]}],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":24,"snaps":[{"begin":2,"length":1}]},{"pool":25,"snaps":[{"begin":4,"length":2},{"begin":8,"length":2},{"begin":12,"length":1}]},{"pool":29,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":30,"snaps":[{"begin":3,"length":1},{"begin":5,"length":1}]},{"pool":31,"snaps":[{"begin":2,"length":1}]},{"pool":32,"snaps":[{"begin":3,"length":1}]},{"pool":34,"snaps":[{"begin":3,"length":1}]},{"pool":36,"snaps":[{"begin":3,"length":1}]},{"pool":38,"snaps":[{"begin":3,"length":1}]},{"pool":39,"snaps":[{"begin":2,"length":1}]},{"pool":40,"snaps":[{"begin":3,"length":1}]},{"pool":41,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":42,"snaps":[{"begin":5,"length":1}]},{"pool":43,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":44,"snaps":[{"begin":5,"length":1}]},{"pool":45,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":46,"snaps":[{"begin":5,"length":1}]},{"pool":47,"snaps":[{"begin":4,"length":1},{"begin":6,"length":1}]},{"pool":48,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":49,"snaps":[{"begin":4,"length":1},{"begin":6,"length":1}]},{"pool":50,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":51,"snaps":[{"begin":6,"length":1}]},{"pool":52,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":53,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":54,"snaps":[{"begin":5,"length":1}]},{"pool":55,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":56,"snaps":[{"begin":5,"length":1}]},{"pool":57,"snaps":[{"begin":5,"length":1}]},{"pool":58,"snaps":[{"begin":6,"length":1}]},{"pool":59,"snaps":[{"begin":2,"length":1},{"begin":5,"length":1}]},{"pool":60,"snaps":[{"begin":6,"length":1}]},{"pool":61,"snaps":[{"begin":4,"length":2}]},{"pool":62,"snaps":[{"begin":5,"length":2}]},{"pool":63,"snaps":[{"begin":5,"length":1}]},{"pool":64,"snaps":[{"begin":6,"length":1}]},{"pool":65,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":66,"snaps":[{"begin":5,"length":1}]},{"pool":69,"snaps":[{"begin":2,"length":1}]},{"pool":70,"snaps":[{"begin":3,"length":1}]},{"pool":71,"snaps":[{"begin":2,"length":1}]},{"pool":72,"snaps":[{"begin":3,"length":1}]},{"pool":74,"snaps":[{"begin":3,"length":1},{"begin":5,"length":1}]},{"pool":76,"snaps":[{"begin":3,"length":1}]},{"pool":78,"snaps":[{"begin":3,"length":1}]},{"pool":79,"snaps":[{"begin":2,"length":1}]},{"pool":80,"snaps":[{"begin":3,"length":1}]},{"pool":81,"snaps":[{"begin":2,"length":1}]},{"pool":82,"snaps":[{"begin":3,"length":1}]},{"pool":83,"snaps":[{"begin":2,"length":1}]},{"pool":84,"snaps":[{"begin":5,"length":1}]},{"pool":85,"snaps":[{"begin":4,"length":1}]},{"pool":86,"snaps":[{"begin":5,"length":2}]},{"pool":87,"snaps":[{"begin":4,"length":1}]},{"pool":88,"snaps":[{"begin":5,"length":2}]},{"pool":89,"snaps":[{"begin":4,"length":1}]},{"pool":90,"snaps":[{"begin":5,"length":2}]},{"pool":91,"snaps":[{"begin":5,"length":2}]},{"pool":92,"snaps":[{"begin":5,"length":2}]},{"pool":93,"snaps":[{"begin":2,"length":1}]},{"pool":94,"snaps":[{"begin":5,"length":1}]},{"pool":96,"snaps":[{"begin":3,"length":1}]},{"pool":97,"snaps":[{"begin":4,"length":1}]},{"pool":98,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":99,"snaps":[{"begin":4,"length":1}]},{"pool":100,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":101,"snaps":[{"begin":4,"length":2}]},{"pool":102,"snaps":[{"begin":5,"length":3}]},{"pool":103,"snaps":[{"begin":4,"length":1},{"begin":6,"length":1}]},{"pool":104,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":105,"snaps":[{"begin":4,"length":1}]},{"pool":106,"snaps":[{"begin":5,"length":2}]},{"pool":109,"snaps":[{"begin":2,"length":1}]},{"pool":110,"snaps":[{"begin":3,"length":1}]},{"pool":111,"snaps":[{"begin":2,"length":1}]},{"pool":112,"snaps":[{"begin":3,"length":1}]},{"pool":114,"snaps":[{"begin":3,"length":1},{"begin":5,"length":1}]},{"pool":116,"snaps":[{"begin":3,"length":1}]},{"pool":118,"snaps":[{"begin":3,"length":1}]},{"pool":119,"snaps":[{"begin":2,"length":1}]},{"pool":120,"snaps":[{"begin":3,"length":1}]},{"pool":121,"snaps":[{"begin":2,"length":1}]},{"pool":122,"snaps":[{"begin":3,"length":1}]},{"pool":124,"snaps":[{"begin":3,"length":1},{"begin":5,"length":1}]},{"pool":125,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":126,"snaps":[{"begin":5,"length":2}]},{"pool":127,"snaps":[{"begin":4,"length":1}]},{"pool":128,"snaps":[{"begin":5,"length":2}]},{"pool":129,"snaps":[{"begin":4,"length":1}]},{"pool":130,"snaps":[{"begin":5,"length":2}]},{"pool":131,"snaps":[{"begin":5,"length":1}]},{"pool":132,"snaps":[{"begin":5,"length":2}]},{"pool":133,"snaps":[{"begin":2,"length":1}]},{"pool":134,"snaps":[{"begin":3,"length":1},{"begin":5,"length":1}]},{"pool":136,"snaps":[{"begin":3,"length":1}]},{"pool":137,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":138,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":139,"snaps":[{"begin":4,"length":1},{"begin":6,"length":1}]},{"pool":140,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":141,"snaps":[{"begin":4,"length":2}]},{"pool":142,"snaps":[{"begin":5,"length":3}]},{"pool":143,"snaps":[{"begin":4,"length":1},{"begin":6,"length":1}]},{"pool":144,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":145,"snaps":[{"begin":4,"length":1}]},{"pool":146,"snaps":[{"begin":5,"length":2}]},{"pool":149,"snaps":[{"begin":2,"length":1}]},{"pool":150,"snaps":[{"begin":3,"length":1}]},{"pool":151,"snaps":[{"begin":2,"length":1}]},{"pool":152,"snaps":[{"begin":3,"length":1}]},{"pool":154,"snaps":[{"begin":3,"length":1},{"begin":5,"length":1}]},{"pool":156,"snaps":[{"begin":3,"length":1}]},{"pool":158,"snaps":[{"begin":3,"length":1}]},{"pool":159,"snaps":[{"begin":2,"length":1}]},{"pool":160,"snaps":[{"begin":3,"length":1}]},{"pool":161,"snaps":[{"begin":2,"length":1}]},{"pool":162,"snaps":[{"begin":3,"length":1}]},{"pool":163,"snaps":[{"begin":2,"length":1}]},{"pool":164,"snaps":[{"begin":3,"length":1},{"begin":5,"length":1}]},{"pool":165,"snaps":[{"begin":4,"length":1}]},{"pool":166,"snaps":[{"begin":5,"length":2}]},{"pool":167,"snaps":[{"begin":4,"length":1}]},{"pool":168,"snaps":[{"begin":5,"length":2}]},{"pool":169,"snaps":[{"begin":4,"length":1}]},{"pool":170,"snaps":[{"begin":5,"length":2}]},{"pool":171,"snaps":[{"begin":5,"length":1}]},{"pool":172,"snaps":[{"begin":5,"length":2}]},{"pool":173,"snaps":[{"begin":2,"length":1}]},{"pool":174,"snaps":[{"begin":5,"length":1}]},{"pool":175,"snaps":[{"begin":4,"length":2}]},{"pool":176,"snaps":[{"begin":5,"length":2},{"begin":8,"length":1}]},{"pool":177,"snaps":[{"begin":4,"length":1}]},{"pool":178,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":179,"snaps":[{"begin":4,"length":1}]},{"pool":180,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":181,"snaps":[{"begin":4,"length":2}]},{"pool":182,"snaps":[{"begin":5,"length":3}]},{"pool":183,"snaps":[{"begin":4,"length":1}]},{"pool":184,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[{"pool":183,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}}
, 2021-10-27T19:33:03.159 INFO:tasks.ceph:Scrubbing osd.0
, 2021-10-27T19:33:03.160 DEBUG:teuthology.orchestra.run.gibba016:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 config set osd_debug_deep_scrub_sleep 0

Ignoring malparsed backtrace: 2021-10-27T19:33:04.435 INFO:tasks.ceph:pgid 147.1a last_scrub_stamp 2021-10-27T19:27:58.262979+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=27, tm_sec=58, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:33:04.436 INFO:tasks.ceph:pgid 148.1e last_scrub_stamp 2021-10-27T19:28:00.269882+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=28, tm_sec=0, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:33:04.436 INFO:tasks.ceph:pgid 147.19 last_scrub_stamp 2021-10-27T19:27:58.262979+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=27, tm_sec=58, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)

Ignoring malparsed backtrace: 2021-10-27T19:33:24.717 INFO:tasks.ceph:pgid 183.5 last_scrub_stamp 2021-10-27T19:32:40.331087+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=40, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:33:24.717 INFO:tasks.ceph:pgid 183.4 last_scrub_stamp 2021-10-27T19:32:40.331087+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=40, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:33:24.718 INFO:tasks.ceph:pgid 184.c last_scrub_stamp 2021-10-27T19:32:42.336433+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=42, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)

Ignoring malparsed backtrace: 2021-10-27T19:33:45.005 INFO:tasks.ceph:pgid 184.1f last_scrub_stamp 2021-10-27T19:32:42.336433+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=42, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:33:45.005 INFO:tasks.ceph:pgid 184.2 last_scrub_stamp 2021-10-27T19:32:42.336433+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=42, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:33:45.006 INFO:tasks.ceph:pgid 184.6 last_scrub_stamp 2021-10-27T19:32:42.336433+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=42, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)

Ignoring malparsed backtrace: 2021-10-27T19:34:05.285 INFO:tasks.ceph:pgid 183.8 last_scrub_stamp 2021-10-27T19:32:40.331087+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=40, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:34:05.285 INFO:tasks.ceph:pgid 184.c last_scrub_stamp 2021-10-27T19:32:42.336433+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=42, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:34:05.286 INFO:tasks.ceph:pgid 184.d last_scrub_stamp 2021-10-27T19:32:42.336433+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=42, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)

Ignoring malparsed backtrace: 2021-10-27T19:34:45.806 INFO:tasks.ceph:pgid 183.2 last_scrub_stamp 2021-10-27T19:32:40.331087+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=40, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:34:45.807 INFO:tasks.ceph:pgid 184.f last_scrub_stamp 2021-10-27T19:32:42.336433+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=42, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:34:45.807 INFO:tasks.ceph:pgid 148.18 last_scrub_stamp 2021-10-27T19:28:00.269882+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=28, tm_sec=0, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)

Ignoring malparsed backtrace:  ceph version 17.0.0-8454-ged9d0b44 (ed9d0b441738e5d5fee0b1394a629b504365ab69) quincy (dev)
,  1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x152) [0x5ff402c]
,  2: /usr/lib64/ceph/libceph-common.so.2(+0x28524d) [0x5ff424d]

Ignoring malparsed backtrace: ==30436==    by 0x4F00FB0: librados::v14_2_0::RadosClient::connect() (RadosClient.cc:312)
, ==30436==    by 0xFAA207: connect_cluster_pp(librados::v14_2_0::Rados&, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > const&) (test_cxx.cc:161)
, ==30436==    by 0xFAADFF: connect_cluster_pp[abi:cxx11](librados::v14_2_0::Rados&) (test_cxx.cc:126)

Ignoring malparsed backtrace: ==30436==    by 0x938CE24: ??? (in /usr/lib64/libnl-route-3.so.200.26.0)
, ==30436==    by 0x400F8B9: call_init.part.0 (in /usr/lib64/ld-2.28.so)
, ==30436==    by 0x400F9B9: _dl_init (in /usr/lib64/ld-2.28.so)

Ignoring malparsed backtrace: ==30436==    by 0xFCB676: Run (gtest.cc:2848)
, ==30436==    by 0xFCB676: testing::TestInfo::Run() (gtest.cc:2831)
, ==30436==    by 0xFCBC70: Run (gtest.cc:3012)

Ignoring malparsed backtrace: ==30436==    by 0x92C60E: librbd::ImageCtx::ImageCtx(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, char const*, librados::v14_2_0::IoCtx&, bool) (ImageCtx.cc:123)
, ==30436==    by 0x81F4BE: UnknownInlinedFun (ImageCtx.h:239)
, ==30436==    by 0x81F4BE: rbd::mirror::image_replayer::OpenImageRequest<librbd::ImageCtx>::send_open_image() (OpenImageRequest.cc:41)

Ignoring malparsed backtrace: ==30436==    by 0x6064F9A: KeyRing::set_modifier(char const*, char const*, EntityName&, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> > >&) (KeyRing.cc:106)
, ==30436==    by 0x60655DF: KeyRing::decode(ceph::buffer::v15_2_0::list::iterator_impl<true>&) (KeyRing.cc:193)
, ==30436==    by 0x6066474: KeyRing::load(ceph::common::CephContext*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) (KeyRing.cc:219)

Ignoring malparsed backtrace: ==30436==    by 0x6064BA2: encode (encoding.h:213)
, ==30436==    by 0x6064BA2: encode (encoding.h:219)
, ==30436==    by 0x6064BA2: KeyRing::set_modifier(char const*, char const*, EntityName&, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::v15_2_0::list> > >&) (KeyRing.cc:117)

Ignoring malparsed backtrace: 2021-10-27T19:33:02.148 INFO:teuthology.orchestra.run.gibba016.stdout:{"epoch":1328,"fsid":"5a830528-8f3d-4eb2-add7-9b2182a4cd3c","created":"2021-10-27T19:09:35.021117+0000","modified":"2021-10-27T19:33:00.308113+0000","last_up_change":"2021-10-27T19:09:54.312852+0000","last_in_change":"2021-10-27T19:09:35.756621+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":184,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2021-10-27T19:09:54.736136+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2021-10-27T19:09:56.157326+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":147,"pool_name":"test-rbd-mirror-gibba016-30436-145","create_time":"2021-10-27T19:27:57.881233+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1029","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":148,"pool_name":"test-rbd-mirror-gibba016-30436-146","create_time":"2021-10-27T19:27:59.285331+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1031","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":183,"pool_name":"test-rados-api-gibba016-30436-181","create_time":"2021-10-27T19:32:39.980131+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1326","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":8,"snap_epoch":1326,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":184,"pool_name":"test-rados-api-gibba016-30436-182","create_time":"2021-10-27T19:32:42.015126+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1327","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":10,"snap_epoch":1327,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"a18d1ceb-edb8-41f1-803d-f4480b2941ed","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":1311,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6802","nonce":28946},{"type":"v1","addr":"172.21.2.116:6804","nonce":28946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6806","nonce":28946},{"type":"v1","addr":"172.21.2.116:6808","nonce":28946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6814","nonce":28946},{"type":"v1","addr":"172.21.2.116:6815","nonce":28946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6810","nonce":28946},{"type":"v1","addr":"172.21.2.116:6813","nonce":28946}]},"public_addr":"172.21.2.116:6804/28946","cluster_addr":"172.21.2.116:6808/28946","heartbeat_back_addr":"172.21.2.116:6815/28946","heartbeat_front_addr":"172.21.2.116:6813/28946","state":["exists","up"]},{"osd":1,"uuid":"7545bf9e-4f37-4aa5-b2dc-d5994995e31c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":1311,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6800","nonce":28947},{"type":"v1","addr":"172.21.2.116:6801","nonce":28947}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6803","nonce":28947},{"type":"v1","addr":"172.21.2.116:6805","nonce":28947}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6811","nonce":28947},{"type":"v1","addr":"172.21.2.116:6812","nonce":28947}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6807","nonce":28947},{"type":"v1","addr":"172.21.2.116:6809","nonce":28947}]},"public_addr":"172.21.2.116:6801/28947","cluster_addr":"172.21.2.116:6805/28947","heartbeat_back_addr":"172.21.2.116:6812/28947","heartbeat_front_addr":"172.21.2.116:6809/28947","state":["exists","up"]},{"osd":2,"uuid":"c1f6b3b1-c86b-449f-aacf-d757769f5caf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":1311,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6816","nonce":28948},{"type":"v1","addr":"172.21.2.116:6817","nonce":28948}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6818","nonce":28948},{"type":"v1","addr":"172.21.2.116:6819","nonce":28948}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6822","nonce":28948},{"type":"v1","addr":"172.21.2.116:6823","nonce":28948}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.116:6820","nonce":28948},{"type":"v1","addr":"172.21.2.116:6821","nonce":28948}]},"public_addr":"172.21.2.116:6817/28948","cluster_addr":"172.21.2.116:6819/28948","heartbeat_back_addr":"172.21.2.116:6823/28948","heartbeat_front_addr":"172.21.2.116:6821/28948","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-10-27T19:09:36.834954+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-10-27T19:09:36.848998+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-10-27T19:09:36.848327+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"147.5","mappings":[{"from":2,"to":0}]},{"pgid":"147.1b","mappings":[{"from":0,"to":1}]},{"pgid":"147.1e","mappings":[{"from":2,"to":1}]}],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":24,"snaps":[{"begin":2,"length":1}]},{"pool":25,"snaps":[{"begin":4,"length":2},{"begin":8,"length":2},{"begin":12,"length":1}]},{"pool":29,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":30,"snaps":[{"begin":3,"length":1},{"begin":5,"length":1}]},{"pool":31,"snaps":[{"begin":2,"length":1}]},{"pool":32,"snaps":[{"begin":3,"length":1}]},{"pool":34,"snaps":[{"begin":3,"length":1}]},{"pool":36,"snaps":[{"begin":3,"length":1}]},{"pool":38,"snaps":[{"begin":3,"length":1}]},{"pool":39,"snaps":[{"begin":2,"length":1}]},{"pool":40,"snaps":[{"begin":3,"length":1}]},{"pool":41,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":42,"snaps":[{"begin":5,"length":1}]},{"pool":43,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":44,"snaps":[{"begin":5,"length":1}]},{"pool":45,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":46,"snaps":[{"begin":5,"length":1}]},{"pool":47,"snaps":[{"begin":4,"length":1},{"begin":6,"length":1}]},{"pool":48,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":49,"snaps":[{"begin":4,"length":1},{"begin":6,"length":1}]},{"pool":50,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":51,"snaps":[{"begin":6,"length":1}]},{"pool":52,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":53,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":54,"snaps":[{"begin":5,"length":1}]},{"pool":55,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":56,"snaps":[{"begin":5,"length":1}]},{"pool":57,"snaps":[{"begin":5,"length":1}]},{"pool":58,"snaps":[{"begin":6,"length":1}]},{"pool":59,"snaps":[{"begin":2,"length":1},{"begin":5,"length":1}]},{"pool":60,"snaps":[{"begin":6,"length":1}]},{"pool":61,"snaps":[{"begin":4,"length":2}]},{"pool":62,"snaps":[{"begin":5,"length":2}]},{"pool":63,"snaps":[{"begin":5,"length":1}]},{"pool":64,"snaps":[{"begin":6,"length":1}]},{"pool":65,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":66,"snaps":[{"begin":5,"length":1}]},{"pool":69,"snaps":[{"begin":2,"length":1}]},{"pool":70,"snaps":[{"begin":3,"length":1}]},{"pool":71,"snaps":[{"begin":2,"length":1}]},{"pool":72,"snaps":[{"begin":3,"length":1}]},{"pool":74,"snaps":[{"begin":3,"length":1},{"begin":5,"length":1}]},{"pool":76,"snaps":[{"begin":3,"length":1}]},{"pool":78,"snaps":[{"begin":3,"length":1}]},{"pool":79,"snaps":[{"begin":2,"length":1}]},{"pool":80,"snaps":[{"begin":3,"length":1}]},{"pool":81,"snaps":[{"begin":2,"length":1}]},{"pool":82,"snaps":[{"begin":3,"length":1}]},{"pool":83,"snaps":[{"begin":2,"length":1}]},{"pool":84,"snaps":[{"begin":5,"length":1}]},{"pool":85,"snaps":[{"begin":4,"length":1}]},{"pool":86,"snaps":[{"begin":5,"length":2}]},{"pool":87,"snaps":[{"begin":4,"length":1}]},{"pool":88,"snaps":[{"begin":5,"length":2}]},{"pool":89,"snaps":[{"begin":4,"length":1}]},{"pool":90,"snaps":[{"begin":5,"length":2}]},{"pool":91,"snaps":[{"begin":5,"length":2}]},{"pool":92,"snaps":[{"begin":5,"length":2}]},{"pool":93,"snaps":[{"begin":2,"length":1}]},{"pool":94,"snaps":[{"begin":5,"length":1}]},{"pool":96,"snaps":[{"begin":3,"length":1}]},{"pool":97,"snaps":[{"begin":4,"length":1}]},{"pool":98,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":99,"snaps":[{"begin":4,"length":1}]},{"pool":100,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":101,"snaps":[{"begin":4,"length":2}]},{"pool":102,"snaps":[{"begin":5,"length":3}]},{"pool":103,"snaps":[{"begin":4,"length":1},{"begin":6,"length":1}]},{"pool":104,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":105,"snaps":[{"begin":4,"length":1}]},{"pool":106,"snaps":[{"begin":5,"length":2}]},{"pool":109,"snaps":[{"begin":2,"length":1}]},{"pool":110,"snaps":[{"begin":3,"length":1}]},{"pool":111,"snaps":[{"begin":2,"length":1}]},{"pool":112,"snaps":[{"begin":3,"length":1}]},{"pool":114,"snaps":[{"begin":3,"length":1},{"begin":5,"length":1}]},{"pool":116,"snaps":[{"begin":3,"length":1}]},{"pool":118,"snaps":[{"begin":3,"length":1}]},{"pool":119,"snaps":[{"begin":2,"length":1}]},{"pool":120,"snaps":[{"begin":3,"length":1}]},{"pool":121,"snaps":[{"begin":2,"length":1}]},{"pool":122,"snaps":[{"begin":3,"length":1}]},{"pool":124,"snaps":[{"begin":3,"length":1},{"begin":5,"length":1}]},{"pool":125,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":126,"snaps":[{"begin":5,"length":2}]},{"pool":127,"snaps":[{"begin":4,"length":1}]},{"pool":128,"snaps":[{"begin":5,"length":2}]},{"pool":129,"snaps":[{"begin":4,"length":1}]},{"pool":130,"snaps":[{"begin":5,"length":2}]},{"pool":131,"snaps":[{"begin":5,"length":1}]},{"pool":132,"snaps":[{"begin":5,"length":2}]},{"pool":133,"snaps":[{"begin":2,"length":1}]},{"pool":134,"snaps":[{"begin":3,"length":1},{"begin":5,"length":1}]},{"pool":136,"snaps":[{"begin":3,"length":1}]},{"pool":137,"snaps":[{"begin":2,"length":1},{"begin":4,"length":1}]},{"pool":138,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":139,"snaps":[{"begin":4,"length":1},{"begin":6,"length":1}]},{"pool":140,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":141,"snaps":[{"begin":4,"length":2}]},{"pool":142,"snaps":[{"begin":5,"length":3}]},{"pool":143,"snaps":[{"begin":4,"length":1},{"begin":6,"length":1}]},{"pool":144,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":145,"snaps":[{"begin":4,"length":1}]},{"pool":146,"snaps":[{"begin":5,"length":2}]},{"pool":149,"snaps":[{"begin":2,"length":1}]},{"pool":150,"snaps":[{"begin":3,"length":1}]},{"pool":151,"snaps":[{"begin":2,"length":1}]},{"pool":152,"snaps":[{"begin":3,"length":1}]},{"pool":154,"snaps":[{"begin":3,"length":1},{"begin":5,"length":1}]},{"pool":156,"snaps":[{"begin":3,"length":1}]},{"pool":158,"snaps":[{"begin":3,"length":1}]},{"pool":159,"snaps":[{"begin":2,"length":1}]},{"pool":160,"snaps":[{"begin":3,"length":1}]},{"pool":161,"snaps":[{"begin":2,"length":1}]},{"pool":162,"snaps":[{"begin":3,"length":1}]},{"pool":163,"snaps":[{"begin":2,"length":1}]},{"pool":164,"snaps":[{"begin":3,"length":1},{"begin":5,"length":1}]},{"pool":165,"snaps":[{"begin":4,"length":1}]},{"pool":166,"snaps":[{"begin":5,"length":2}]},{"pool":167,"snaps":[{"begin":4,"length":1}]},{"pool":168,"snaps":[{"begin":5,"length":2}]},{"pool":169,"snaps":[{"begin":4,"length":1}]},{"pool":170,"snaps":[{"begin":5,"length":2}]},{"pool":171,"snaps":[{"begin":5,"length":1}]},{"pool":172,"snaps":[{"begin":5,"length":2}]},{"pool":173,"snaps":[{"begin":2,"length":1}]},{"pool":174,"snaps":[{"begin":5,"length":1}]},{"pool":175,"snaps":[{"begin":4,"length":2}]},{"pool":176,"snaps":[{"begin":5,"length":2},{"begin":8,"length":1}]},{"pool":177,"snaps":[{"begin":4,"length":1}]},{"pool":178,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":179,"snaps":[{"begin":4,"length":1}]},{"pool":180,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]},{"pool":181,"snaps":[{"begin":4,"length":2}]},{"pool":182,"snaps":[{"begin":5,"length":3}]},{"pool":183,"snaps":[{"begin":4,"length":1}]},{"pool":184,"snaps":[{"begin":5,"length":1},{"begin":7,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[{"pool":183,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}}
, 2021-10-27T19:33:03.159 INFO:tasks.ceph:Scrubbing osd.0
, 2021-10-27T19:33:03.160 DEBUG:teuthology.orchestra.run.gibba016:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 config set osd_debug_deep_scrub_sleep 0

Ignoring malparsed backtrace: 2021-10-27T19:33:04.435 INFO:tasks.ceph:pgid 147.1a last_scrub_stamp 2021-10-27T19:27:58.262979+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=27, tm_sec=58, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:33:04.436 INFO:tasks.ceph:pgid 148.1e last_scrub_stamp 2021-10-27T19:28:00.269882+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=28, tm_sec=0, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:33:04.436 INFO:tasks.ceph:pgid 147.19 last_scrub_stamp 2021-10-27T19:27:58.262979+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=27, tm_sec=58, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)

Ignoring malparsed backtrace: 2021-10-27T19:33:24.717 INFO:tasks.ceph:pgid 183.5 last_scrub_stamp 2021-10-27T19:32:40.331087+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=40, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:33:24.717 INFO:tasks.ceph:pgid 183.4 last_scrub_stamp 2021-10-27T19:32:40.331087+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=40, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:33:24.718 INFO:tasks.ceph:pgid 184.c last_scrub_stamp 2021-10-27T19:32:42.336433+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=42, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)

Ignoring malparsed backtrace: 2021-10-27T19:33:45.005 INFO:tasks.ceph:pgid 184.1f last_scrub_stamp 2021-10-27T19:32:42.336433+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=42, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:33:45.005 INFO:tasks.ceph:pgid 184.2 last_scrub_stamp 2021-10-27T19:32:42.336433+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=42, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:33:45.006 INFO:tasks.ceph:pgid 184.6 last_scrub_stamp 2021-10-27T19:32:42.336433+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=42, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)

Ignoring malparsed backtrace: 2021-10-27T19:34:05.285 INFO:tasks.ceph:pgid 183.8 last_scrub_stamp 2021-10-27T19:32:40.331087+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=40, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:34:05.285 INFO:tasks.ceph:pgid 184.c last_scrub_stamp 2021-10-27T19:32:42.336433+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=42, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:34:05.286 INFO:tasks.ceph:pgid 184.d last_scrub_stamp 2021-10-27T19:32:42.336433+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=42, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)

Ignoring malparsed backtrace: 2021-10-27T19:34:45.806 INFO:tasks.ceph:pgid 183.2 last_scrub_stamp 2021-10-27T19:32:40.331087+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=40, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:34:45.807 INFO:tasks.ceph:pgid 184.f last_scrub_stamp 2021-10-27T19:32:42.336433+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=32, tm_sec=42, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)
, 2021-10-27T19:34:45.807 INFO:tasks.ceph:pgid 148.18 last_scrub_stamp 2021-10-27T19:28:00.269882+0000 time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=28, tm_sec=0, tm_wday=2, tm_yday=300, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=10, tm_mday=27, tm_hour=19, tm_min=33, tm_sec=2, tm_wday=2, tm_yday=300, tm_isdst=0)

Found 7 distinct failure reasons
Failure: "2021-10-27T11:39:09.436515+0000 mon.a (mon.0) 95 : cluster [WRN] Health check failed: Degraded data redundancy: 2/4 objects degraded (50.000%), 1 pg degraded (PG_DEGRADED)" in cluster log
8 jobs: ['6459610', '6459667', '6459743', '6459680', '6459674', '6459735', '6459703', '6459665']
suites intersection: ['openstack}']
suites union: ['clients/mirror', 'clients/{mirror-extra', 'cluster/{2-node', 'clusters/{fixed-1', 'clusters/{fixed-3', 'config/copy-on-read', 'features/defaults', 'features/format-1', 'features/readbalance', 'min-compat-client/octopus', 'mirror}', 'msgr-failures/few', 'objectstore/bluestore-comp-lz4', 'objectstore/bluestore-comp-snappy', 'objectstore/filestore-xfs', 'openstack}', 'policy/simple', 'pool/ceph_and_immutable_object_cache', 'pool/none', 'pool/replicated-data-pool', 'pool/small-cache-pool', 'rbd-mirror/four-per-cluster', 'rbd/cli/{base/install', 'rbd/cli_v1/{base/install', 'rbd/immutable-object-cache/{clusters/{fix-2', 'rbd/librbd/{cache/writearound', 'rbd/mirror-thrash/{base/install', 'rbd/mirror/{base/install', 'rbd/qemu/{cache/writearound', 'supported-random-distro$/{centos_8.stream}', 'supported-random-distro$/{centos_8}', 'supported-random-distro$/{rhel_8}', 'supported-random-distro$/{ubuntu_latest}', 'workloads/c_api_tests_with_defaults}', 'workloads/c_api_tests_with_journaling}', 'workloads/qemu_xfstests}', 'workloads/rbd-mirror-fsx-workunit}', 'workloads/rbd-mirror-snapshot-workunit-minimum}', 'workloads/rbd-mirror-stress-workunit-min-compat-client-octopus}', 'workloads/rbd_cli_import_export}', 'workloads/rbd_cli_migration}']

Failure: reached maximum tries (90) after waiting for 540 seconds
3 jobs: ['6459734', '6459742', '6459663']
suites intersection: ['msgr-failures/few', 'openstack}']
suites union: ['clusters/{fixed-2', 'clusters/{fixed-3', 'features/defaults', 'features/journaling', 'msgr-failures/few', 'objectstore/bluestore-comp-zlib', 'objectstore/bluestore-comp-zstd', 'objectstore/filestore-xfs', 'openstack}', 'pool/ec-cache-pool', 'pool/replicated-data-pool', 'rbd/encryption/{cache/writearound', 'rbd/qemu/{cache/writearound', 'rbd/thrash/{base/install', 'supported-random-distro$/{rhel_8}', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/default', 'thrashosds-health', 'workloads/qemu_fsstress}', 'workloads/qemu_xfstests_luks2}', 'workloads/rbd_fsx_rate_limit}']

Timeout 3h running clone.client.0/qa/workunits/rbd/cli_generic.sh
5 jobs: ['6459715', '6459660', '6459594', '6459752', '6459688']
suites intersection: ['clusters/{fixed-1', 'features/format-1', 'msgr-failures/few', 'openstack}', 'pool/small-cache-pool', 'rbd/cli_v1/{base/install', 'workloads/rbd_cli_generic}']
suites union: ['clusters/{fixed-1', 'features/format-1', 'msgr-failures/few', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp-lz4', 'objectstore/bluestore-comp-zstd', 'objectstore/bluestore-low-osd-mem-target', 'objectstore/filestore-xfs', 'openstack}', 'pool/small-cache-pool', 'rbd/cli_v1/{base/install', 'supported-random-distro$/{centos_8}', 'supported-random-distro$/{rhel_8}', 'supported-random-distro$/{ubuntu_latest}', 'workloads/rbd_cli_generic}']

Timeout 3h running clone.client.0/qa/workunits/rbd/qemu_dynamic_features.sh
1 jobs: ['6459598']
suites: ['clusters/{fixed-3', 'objectstore/bluestore-stupid', 'openstack}', 'qemu/xfstests', 'rbd/maintenance/{base/install', 'supported-random-distro$/{rhel_8}', 'workloads/dynamic_features}']

Failure: Command failed (workunit test rbd/test_rbd_mirror.sh) on gibba016 with status 134: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=ed9d0b441738e5d5fee0b1394a629b504365ab69 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 VALGRIND=\'--tool=memcheck --leak-check=full\' adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/test_rbd_mirror.sh'
1 jobs: ['6459670']
suites: ['centos_latest', 'clusters/{fixed-1', 'objectstore/filestore-xfs', 'openstack}', 'rbd/valgrind/{base/install', 'validator/memcheck', 'workloads/rbd_mirror}']

Dead: b'2021-10-28T04:15:50.797 DEBUG:teuthology.exit:Finished running handlers'
ceph version 17.0.0-8454-ged9d0b44 (ed9d0b441738e5d5fee0b1394a629b504365ab69) quincy (dev)
 1: /lib64/libpthread.so.0(+0x12b20) [0x7fbad437eb20]
 2: gsignal()
 3: abort()
 4: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x1b8) [0x55f198af8504]
 5: (KernelDevice::_aio_thread()+0xf9e) [0x55f19971ca3e]
 6: (KernelDevice::AioCompletionThread::entry()+0x11) [0x55f199728691]
 7: /lib64/libpthread.so.0(+0x814a) [0x7fbad437414a]
 8: clone()
2 jobs: ['6459653', '6459636']
suites intersection: ['clients/mirror', 'cluster/{2-node', 'msgr-failures/few', 'openstack}', 'rbd-mirror/four-per-cluster', 'rbd/mirror-thrash/{base/install']
suites union: ['clients/mirror', 'cluster/{2-node', 'msgr-failures/few', 'objectstore/bluestore-low-osd-mem-target', 'objectstore/bluestore-stupid', 'openstack}', 'policy/none', 'policy/simple', 'rbd-mirror/four-per-cluster', 'rbd/mirror-thrash/{base/install', 'supported-random-distro$/{centos_8.stream}', 'supported-random-distro$/{centos_8}', 'workloads/rbd-mirror-snapshot-stress-workunit-exclusive-lock}', 'workloads/rbd-mirror-snapshot-stress-workunit-fast-diff}']

Failure: Command failed on gibba016 with status 1: 'test -n "$(ls /tmp/ceph-immutable-object-cache )"'
1 jobs: ['6459684']
suites: ['openstack}', 'pool/ceph_and_immutable_object_cache', 'rbd/immutable-object-cache/{clusters/{fix-2', 'supported-random-distro$/{centos_8}', 'workloads/qemu_on_immutable_object_cache_and_thrash}']