2022-09-08T08:06:23.087 INFO:root:teuthology version: 1.1.1.dev418+gb58f2c18 2022-09-08T08:06:23.089 INFO:teuthology.results:seed: '9852' 2022-09-08T08:08:45.300 INFO:root:teuthology version: 1.1.1.dev418+gb58f2c18 2022-09-08T08:08:45.304 INFO:teuthology.results:Waiting up to 43200 seconds for tests to finish... 2022-09-08T20:09:40.154 WARNING:teuthology.results:test(s) did not finish before timeout of 43200 seconds 2022-09-08T20:09:40.448 INFO:scrape:Found 3 jobs 2022-09-08T20:09:40.964 WARNING:scrape:Ignoring malparsed backtrace: ceph version 17.0.0-14859-ge85e1217 (e85e121747baf9691b263effef4c284a96431457) quincy (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7fa3a1a50cb0] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x229) [0x5598e2f9d04b] 2022-09-08T20:09:40.968 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T08:24:35.799 INFO:teuthology.orchestra.run.smithi085.stdout:{"epoch":51,"fsid":"11b9b9ed-354a-4cb5-853e-04e8678930c4","created":"2022-09-08T08:22:54.094684+0000","modified":"2022-09-08T08:24:15.331804+0000","last_up_change":"2022-09-08T08:24:04.259680+0000","last_in_change":"2022-09-08T08:22:55.340984+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-08T08:22:59.695319+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-09-08T08:23:03.574393+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-09-08T08:23:10.014014+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"6bf674ea-469d-492a-b3b6-94dfabcd2b5d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":24,"down_at":40,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6801","nonce":38290}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6800","nonce":38290}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6805","nonce":38290}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6804","nonce":38290}]},"public_addr":"172.21.15.85:6801/38290","cluster_addr":"172.21.15.85:6800/38290","heartbeat_back_addr":"172.21.15.85:6805/38290","heartbeat_front_addr":"172.21.15.85:6804/38290","state":["exists"]},{"osd":1,"uuid":"f496fa03-3aa3-4f1c-a182-33a22ac73734","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6803","nonce":38317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6802","nonce":38317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6806","nonce":38317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6807","nonce":38317}]},"public_addr":"172.21.15.85:6803/38317","cluster_addr":"172.21.15.85:6802/38317","heartbeat_back_addr":"172.21.15.85:6806/38317","heartbeat_front_addr":"172.21.15.85:6807/38317","state":["exists","up"]},{"osd":2,"uuid":"e78d76bc-7d8d-4f3b-8e7b-bb873df19bad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":40,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6800","nonce":38150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6801","nonce":38150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6804","nonce":38150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6805","nonce":38150}]},"public_addr":"172.21.15.93:6800/38150","cluster_addr":"172.21.15.93:6801/38150","heartbeat_back_addr":"172.21.15.93:6804/38150","heartbeat_front_addr":"172.21.15.93:6805/38150","state":["exists","up"]},{"osd":3,"uuid":"16ef1dba-c6a5-4ed2-959b-4f515867860d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6803","nonce":38179}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6802","nonce":38179}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6807","nonce":38179}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6806","nonce":38179}]},"public_addr":"172.21.15.93:6803/38179","cluster_addr":"172.21.15.93:6802/38179","heartbeat_back_addr":"172.21.15.93:6807/38179","heartbeat_front_addr":"172.21.15.93:6806/38179","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2022-09-08T08:24:04.259680+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[{"pgid":"3.2","osds":[1,0,2]},{"pgid":"3.a","osds":[3,0,1]}],"pg_upmap_items":[{"pgid":"3.a","mappings":[{"from":1,"to":3},{"from":2,"to":0}]}],"pg_temp":[{"pgid":"3.2","osds":[1,2,3]},{"pgid":"3.a","osds":[3,1,2]}],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2022-09-08T08:24:38.814 DEBUG:teuthology.orchestra.run.smithi085:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2022-09-08T08:24:39.154 INFO:teuthology.orchestra.run.smithi085.stdout: 2022-09-08T20:09:40.970 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T08:26:29.861 INFO:teuthology.orchestra.run.smithi085.stdout: , 2022-09-08T08:26:29.862 INFO:teuthology.orchestra.run.smithi085.stdout:{"epoch":51,"fsid":"11b9b9ed-354a-4cb5-853e-04e8678930c4","created":"2022-09-08T08:22:54.094684+0000","modified":"2022-09-08T08:24:15.331804+0000","last_up_change":"2022-09-08T08:24:04.259680+0000","last_in_change":"2022-09-08T08:22:55.340984+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-08T08:22:59.695319+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-09-08T08:23:03.574393+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-09-08T08:23:10.014014+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"6bf674ea-469d-492a-b3b6-94dfabcd2b5d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":24,"down_at":40,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6801","nonce":38290}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6800","nonce":38290}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6805","nonce":38290}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6804","nonce":38290}]},"public_addr":"172.21.15.85:6801/38290","cluster_addr":"172.21.15.85:6800/38290","heartbeat_back_addr":"172.21.15.85:6805/38290","heartbeat_front_addr":"172.21.15.85:6804/38290","state":["exists"]},{"osd":1,"uuid":"f496fa03-3aa3-4f1c-a182-33a22ac73734","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6803","nonce":38317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6802","nonce":38317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6806","nonce":38317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6807","nonce":38317}]},"public_addr":"172.21.15.85:6803/38317","cluster_addr":"172.21.15.85:6802/38317","heartbeat_back_addr":"172.21.15.85:6806/38317","heartbeat_front_addr":"172.21.15.85:6807/38317","state":["exists","up"]},{"osd":2,"uuid":"e78d76bc-7d8d-4f3b-8e7b-bb873df19bad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":40,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6800","nonce":38150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6801","nonce":38150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6804","nonce":38150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6805","nonce":38150}]},"public_addr":"172.21.15.93:6800/38150","cluster_addr":"172.21.15.93:6801/38150","heartbeat_back_addr":"172.21.15.93:6804/38150","heartbeat_front_addr":"172.21.15.93:6805/38150","state":["exists","up"]},{"osd":3,"uuid":"16ef1dba-c6a5-4ed2-959b-4f515867860d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6803","nonce":38179}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6802","nonce":38179}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6807","nonce":38179}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6806","nonce":38179}]},"public_addr":"172.21.15.93:6803/38179","cluster_addr":"172.21.15.93:6802/38179","heartbeat_back_addr":"172.21.15.93:6807/38179","heartbeat_front_addr":"172.21.15.93:6806/38179","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2022-09-08T08:24:04.259680+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[{"pgid":"3.2","osds":[1,0,2]},{"pgid":"3.a","osds":[3,0,1]}],"pg_upmap_items":[{"pgid":"3.a","mappings":[{"from":1,"to":3},{"from":2,"to":0}]}],"pg_temp":[{"pgid":"3.2","osds":[1,2,3]},{"pgid":"3.a","osds":[3,1,2]}],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2022-09-08T08:26:32.875 DEBUG:teuthology.orchestra.run.smithi085:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2022-09-08T20:09:40.972 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T08:28:23.507 DEBUG:teuthology.orchestra.run.smithi085:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2022-09-08T08:28:23.841 INFO:teuthology.orchestra.run.smithi085.stdout: , 2022-09-08T08:28:23.842 INFO:teuthology.orchestra.run.smithi085.stdout:{"epoch":51,"fsid":"11b9b9ed-354a-4cb5-853e-04e8678930c4","created":"2022-09-08T08:22:54.094684+0000","modified":"2022-09-08T08:24:15.331804+0000","last_up_change":"2022-09-08T08:24:04.259680+0000","last_in_change":"2022-09-08T08:22:55.340984+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-08T08:22:59.695319+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-09-08T08:23:03.574393+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-09-08T08:23:10.014014+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"6bf674ea-469d-492a-b3b6-94dfabcd2b5d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":24,"down_at":40,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6801","nonce":38290}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6800","nonce":38290}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6805","nonce":38290}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6804","nonce":38290}]},"public_addr":"172.21.15.85:6801/38290","cluster_addr":"172.21.15.85:6800/38290","heartbeat_back_addr":"172.21.15.85:6805/38290","heartbeat_front_addr":"172.21.15.85:6804/38290","state":["exists"]},{"osd":1,"uuid":"f496fa03-3aa3-4f1c-a182-33a22ac73734","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6803","nonce":38317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6802","nonce":38317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6806","nonce":38317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6807","nonce":38317}]},"public_addr":"172.21.15.85:6803/38317","cluster_addr":"172.21.15.85:6802/38317","heartbeat_back_addr":"172.21.15.85:6806/38317","heartbeat_front_addr":"172.21.15.85:6807/38317","state":["exists","up"]},{"osd":2,"uuid":"e78d76bc-7d8d-4f3b-8e7b-bb873df19bad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":40,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6800","nonce":38150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6801","nonce":38150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6804","nonce":38150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6805","nonce":38150}]},"public_addr":"172.21.15.93:6800/38150","cluster_addr":"172.21.15.93:6801/38150","heartbeat_back_addr":"172.21.15.93:6804/38150","heartbeat_front_addr":"172.21.15.93:6805/38150","state":["exists","up"]},{"osd":3,"uuid":"16ef1dba-c6a5-4ed2-959b-4f515867860d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6803","nonce":38179}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6802","nonce":38179}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6807","nonce":38179}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6806","nonce":38179}]},"public_addr":"172.21.15.93:6803/38179","cluster_addr":"172.21.15.93:6802/38179","heartbeat_back_addr":"172.21.15.93:6807/38179","heartbeat_front_addr":"172.21.15.93:6806/38179","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2022-09-08T08:24:04.259680+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[{"pgid":"3.2","osds":[1,0,2]},{"pgid":"3.a","osds":[3,0,1]}],"pg_upmap_items":[{"pgid":"3.a","mappings":[{"from":1,"to":3},{"from":2,"to":0}]}],"pg_temp":[{"pgid":"3.2","osds":[1,2,3]},{"pgid":"3.a","osds":[3,1,2]}],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2022-09-08T20:09:41.346 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T08:30:14.479 INFO:teuthology.orchestra.run.smithi085.stdout:{"epoch":51,"fsid":"11b9b9ed-354a-4cb5-853e-04e8678930c4","created":"2022-09-08T08:22:54.094684+0000","modified":"2022-09-08T08:24:15.331804+0000","last_up_change":"2022-09-08T08:24:04.259680+0000","last_in_change":"2022-09-08T08:22:55.340984+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-08T08:22:59.695319+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-09-08T08:23:03.574393+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-09-08T08:23:10.014014+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"6bf674ea-469d-492a-b3b6-94dfabcd2b5d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":24,"down_at":40,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6801","nonce":38290}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6800","nonce":38290}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6805","nonce":38290}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6804","nonce":38290}]},"public_addr":"172.21.15.85:6801/38290","cluster_addr":"172.21.15.85:6800/38290","heartbeat_back_addr":"172.21.15.85:6805/38290","heartbeat_front_addr":"172.21.15.85:6804/38290","state":["exists"]},{"osd":1,"uuid":"f496fa03-3aa3-4f1c-a182-33a22ac73734","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6803","nonce":38317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6802","nonce":38317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6806","nonce":38317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6807","nonce":38317}]},"public_addr":"172.21.15.85:6803/38317","cluster_addr":"172.21.15.85:6802/38317","heartbeat_back_addr":"172.21.15.85:6806/38317","heartbeat_front_addr":"172.21.15.85:6807/38317","state":["exists","up"]},{"osd":2,"uuid":"e78d76bc-7d8d-4f3b-8e7b-bb873df19bad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":40,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6800","nonce":38150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6801","nonce":38150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6804","nonce":38150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6805","nonce":38150}]},"public_addr":"172.21.15.93:6800/38150","cluster_addr":"172.21.15.93:6801/38150","heartbeat_back_addr":"172.21.15.93:6804/38150","heartbeat_front_addr":"172.21.15.93:6805/38150","state":["exists","up"]},{"osd":3,"uuid":"16ef1dba-c6a5-4ed2-959b-4f515867860d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6803","nonce":38179}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6802","nonce":38179}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6807","nonce":38179}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6806","nonce":38179}]},"public_addr":"172.21.15.93:6803/38179","cluster_addr":"172.21.15.93:6802/38179","heartbeat_back_addr":"172.21.15.93:6807/38179","heartbeat_front_addr":"172.21.15.93:6806/38179","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2022-09-08T08:24:04.259680+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[{"pgid":"3.2","osds":[1,0,2]},{"pgid":"3.a","osds":[3,0,1]}],"pg_upmap_items":[{"pgid":"3.a","mappings":[{"from":1,"to":3},{"from":2,"to":0}]}],"pg_temp":[{"pgid":"3.2","osds":[1,2,3]},{"pgid":"3.a","osds":[3,1,2]}],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2022-09-08T08:30:17.494 DEBUG:teuthology.orchestra.run.smithi085:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2022-09-08T08:30:17.829 INFO:teuthology.orchestra.run.smithi085.stdout: 2022-09-08T20:09:41.349 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T08:32:08.562 INFO:teuthology.orchestra.run.smithi085.stdout: , 2022-09-08T08:32:08.563 INFO:teuthology.orchestra.run.smithi085.stdout:{"epoch":51,"fsid":"11b9b9ed-354a-4cb5-853e-04e8678930c4","created":"2022-09-08T08:22:54.094684+0000","modified":"2022-09-08T08:24:15.331804+0000","last_up_change":"2022-09-08T08:24:04.259680+0000","last_in_change":"2022-09-08T08:22:55.340984+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-08T08:22:59.695319+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-09-08T08:23:03.574393+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-09-08T08:23:10.014014+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"6bf674ea-469d-492a-b3b6-94dfabcd2b5d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":24,"down_at":40,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6801","nonce":38290}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6800","nonce":38290}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6805","nonce":38290}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6804","nonce":38290}]},"public_addr":"172.21.15.85:6801/38290","cluster_addr":"172.21.15.85:6800/38290","heartbeat_back_addr":"172.21.15.85:6805/38290","heartbeat_front_addr":"172.21.15.85:6804/38290","state":["exists"]},{"osd":1,"uuid":"f496fa03-3aa3-4f1c-a182-33a22ac73734","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6803","nonce":38317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6802","nonce":38317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6806","nonce":38317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6807","nonce":38317}]},"public_addr":"172.21.15.85:6803/38317","cluster_addr":"172.21.15.85:6802/38317","heartbeat_back_addr":"172.21.15.85:6806/38317","heartbeat_front_addr":"172.21.15.85:6807/38317","state":["exists","up"]},{"osd":2,"uuid":"e78d76bc-7d8d-4f3b-8e7b-bb873df19bad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":40,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6800","nonce":38150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6801","nonce":38150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6804","nonce":38150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6805","nonce":38150}]},"public_addr":"172.21.15.93:6800/38150","cluster_addr":"172.21.15.93:6801/38150","heartbeat_back_addr":"172.21.15.93:6804/38150","heartbeat_front_addr":"172.21.15.93:6805/38150","state":["exists","up"]},{"osd":3,"uuid":"16ef1dba-c6a5-4ed2-959b-4f515867860d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6803","nonce":38179}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6802","nonce":38179}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6807","nonce":38179}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6806","nonce":38179}]},"public_addr":"172.21.15.93:6803/38179","cluster_addr":"172.21.15.93:6802/38179","heartbeat_back_addr":"172.21.15.93:6807/38179","heartbeat_front_addr":"172.21.15.93:6806/38179","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2022-09-08T08:24:04.259680+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[{"pgid":"3.2","osds":[1,0,2]},{"pgid":"3.a","osds":[3,0,1]}],"pg_upmap_items":[{"pgid":"3.a","mappings":[{"from":1,"to":3},{"from":2,"to":0}]}],"pg_temp":[{"pgid":"3.2","osds":[1,2,3]},{"pgid":"3.a","osds":[3,1,2]}],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2022-09-08T08:32:11.577 DEBUG:teuthology.orchestra.run.smithi085:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2022-09-08T20:09:41.350 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T08:34:02.231 DEBUG:teuthology.orchestra.run.smithi085:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2022-09-08T08:34:02.568 INFO:teuthology.orchestra.run.smithi085.stdout: , 2022-09-08T08:34:02.568 INFO:teuthology.orchestra.run.smithi085.stdout:{"epoch":51,"fsid":"11b9b9ed-354a-4cb5-853e-04e8678930c4","created":"2022-09-08T08:22:54.094684+0000","modified":"2022-09-08T08:24:15.331804+0000","last_up_change":"2022-09-08T08:24:04.259680+0000","last_in_change":"2022-09-08T08:22:55.340984+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-08T08:22:59.695319+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-09-08T08:23:03.574393+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-09-08T08:23:10.014014+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"6bf674ea-469d-492a-b3b6-94dfabcd2b5d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":24,"down_at":40,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6801","nonce":38290}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6800","nonce":38290}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6805","nonce":38290}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6804","nonce":38290}]},"public_addr":"172.21.15.85:6801/38290","cluster_addr":"172.21.15.85:6800/38290","heartbeat_back_addr":"172.21.15.85:6805/38290","heartbeat_front_addr":"172.21.15.85:6804/38290","state":["exists"]},{"osd":1,"uuid":"f496fa03-3aa3-4f1c-a182-33a22ac73734","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6803","nonce":38317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6802","nonce":38317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6806","nonce":38317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6807","nonce":38317}]},"public_addr":"172.21.15.85:6803/38317","cluster_addr":"172.21.15.85:6802/38317","heartbeat_back_addr":"172.21.15.85:6806/38317","heartbeat_front_addr":"172.21.15.85:6807/38317","state":["exists","up"]},{"osd":2,"uuid":"e78d76bc-7d8d-4f3b-8e7b-bb873df19bad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":40,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6800","nonce":38150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6801","nonce":38150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6804","nonce":38150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6805","nonce":38150}]},"public_addr":"172.21.15.93:6800/38150","cluster_addr":"172.21.15.93:6801/38150","heartbeat_back_addr":"172.21.15.93:6804/38150","heartbeat_front_addr":"172.21.15.93:6805/38150","state":["exists","up"]},{"osd":3,"uuid":"16ef1dba-c6a5-4ed2-959b-4f515867860d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6803","nonce":38179}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6802","nonce":38179}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6807","nonce":38179}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6806","nonce":38179}]},"public_addr":"172.21.15.93:6803/38179","cluster_addr":"172.21.15.93:6802/38179","heartbeat_back_addr":"172.21.15.93:6807/38179","heartbeat_front_addr":"172.21.15.93:6806/38179","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2022-09-08T08:24:04.259680+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[{"pgid":"3.2","osds":[1,0,2]},{"pgid":"3.a","osds":[3,0,1]}],"pg_upmap_items":[{"pgid":"3.a","mappings":[{"from":1,"to":3},{"from":2,"to":0}]}],"pg_temp":[{"pgid":"3.2","osds":[1,2,3]},{"pgid":"3.a","osds":[3,1,2]}],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2022-09-08T20:09:41.352 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T08:35:22.554 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~70s , 2022-09-08T08:35:22.735 DEBUG:teuthology.orchestra.run.smithi085:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2022-09-08T08:35:23.075 INFO:teuthology.orchestra.run.smithi085.stdout: 2022-09-08T20:09:41.353 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T08:36:30.191 INFO:teuthology.orchestra.run.smithi085.stdout:{"epoch":54,"fsid":"11b9b9ed-354a-4cb5-853e-04e8678930c4","created":"2022-09-08T08:22:54.094684+0000","modified":"2022-09-08T08:34:11.194702+0000","last_up_change":"2022-09-08T08:24:04.259680+0000","last_in_change":"2022-09-08T08:34:09.186829+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-08T08:22:59.695319+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-09-08T08:23:03.574393+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-09-08T08:23:10.014014+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"6bf674ea-469d-492a-b3b6-94dfabcd2b5d","up":0,"in":0,"weight":0,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":24,"down_at":40,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6801","nonce":38290}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6800","nonce":38290}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6805","nonce":38290}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6804","nonce":38290}]},"public_addr":"172.21.15.85:6801/38290","cluster_addr":"172.21.15.85:6800/38290","heartbeat_back_addr":"172.21.15.85:6805/38290","heartbeat_front_addr":"172.21.15.85:6804/38290","state":["autoout","exists"]},{"osd":1,"uuid":"f496fa03-3aa3-4f1c-a182-33a22ac73734","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6803","nonce":38317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6802","nonce":38317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6806","nonce":38317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6807","nonce":38317}]},"public_addr":"172.21.15.85:6803/38317","cluster_addr":"172.21.15.85:6802/38317","heartbeat_back_addr":"172.21.15.85:6806/38317","heartbeat_front_addr":"172.21.15.85:6807/38317","state":["exists","up"]},{"osd":2,"uuid":"e78d76bc-7d8d-4f3b-8e7b-bb873df19bad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6800","nonce":38150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6801","nonce":38150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6804","nonce":38150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6805","nonce":38150}]},"public_addr":"172.21.15.93:6800/38150","cluster_addr":"172.21.15.93:6801/38150","heartbeat_back_addr":"172.21.15.93:6804/38150","heartbeat_front_addr":"172.21.15.93:6805/38150","state":["exists","up"]},{"osd":3,"uuid":"16ef1dba-c6a5-4ed2-959b-4f515867860d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6803","nonce":38179}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6802","nonce":38179}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6807","nonce":38179}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6806","nonce":38179}]},"public_addr":"172.21.15.93:6803/38179","cluster_addr":"172.21.15.93:6802/38179","heartbeat_back_addr":"172.21.15.93:6807/38179","heartbeat_front_addr":"172.21.15.93:6806/38179","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2022-09-08T08:24:04.259680+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":65536,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[{"pgid":"3.2","osds":[1,0,2]},{"pgid":"3.a","osds":[3,0,1]}],"pg_upmap_items":[{"pgid":"3.a","mappings":[{"from":1,"to":3}]}],"pg_temp":[{"pgid":"3.1","osds":[3,1]},{"pgid":"3.3","osds":[2,3]},{"pgid":"3.4","osds":[2,3]},{"pgid":"3.7","osds":[3,1]},{"pgid":"3.9","osds":[1,2]},{"pgid":"3.b","osds":[3,2]},{"pgid":"3.c","osds":[3,2]},{"pgid":"3.d","osds":[1,3]},{"pgid":"3.f","osds":[1,3]}],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2022-09-08T08:36:32.816 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.2 is failed for ~141s , 2022-09-08T08:36:32.816 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~141s 2022-09-08T20:09:41.354 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T08:37:40.242 DEBUG:teuthology.orchestra.run.smithi085:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2022-09-08T08:37:40.583 INFO:teuthology.orchestra.run.smithi085.stdout: , 2022-09-08T08:37:40.583 INFO:teuthology.orchestra.run.smithi085.stdout:{"epoch":54,"fsid":"11b9b9ed-354a-4cb5-853e-04e8678930c4","created":"2022-09-08T08:22:54.094684+0000","modified":"2022-09-08T08:34:11.194702+0000","last_up_change":"2022-09-08T08:24:04.259680+0000","last_in_change":"2022-09-08T08:34:09.186829+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-08T08:22:59.695319+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-09-08T08:23:03.574393+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-09-08T08:23:10.014014+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"6bf674ea-469d-492a-b3b6-94dfabcd2b5d","up":0,"in":0,"weight":0,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":24,"down_at":40,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6801","nonce":38290}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6800","nonce":38290}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6805","nonce":38290}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6804","nonce":38290}]},"public_addr":"172.21.15.85:6801/38290","cluster_addr":"172.21.15.85:6800/38290","heartbeat_back_addr":"172.21.15.85:6805/38290","heartbeat_front_addr":"172.21.15.85:6804/38290","state":["autoout","exists"]},{"osd":1,"uuid":"f496fa03-3aa3-4f1c-a182-33a22ac73734","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6803","nonce":38317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6802","nonce":38317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6806","nonce":38317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6807","nonce":38317}]},"public_addr":"172.21.15.85:6803/38317","cluster_addr":"172.21.15.85:6802/38317","heartbeat_back_addr":"172.21.15.85:6806/38317","heartbeat_front_addr":"172.21.15.85:6807/38317","state":["exists","up"]},{"osd":2,"uuid":"e78d76bc-7d8d-4f3b-8e7b-bb873df19bad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6800","nonce":38150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6801","nonce":38150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6804","nonce":38150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6805","nonce":38150}]},"public_addr":"172.21.15.93:6800/38150","cluster_addr":"172.21.15.93:6801/38150","heartbeat_back_addr":"172.21.15.93:6804/38150","heartbeat_front_addr":"172.21.15.93:6805/38150","state":["exists","up"]},{"osd":3,"uuid":"16ef1dba-c6a5-4ed2-959b-4f515867860d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6803","nonce":38179}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6802","nonce":38179}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6807","nonce":38179}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6806","nonce":38179}]},"public_addr":"172.21.15.93:6803/38179","cluster_addr":"172.21.15.93:6802/38179","heartbeat_back_addr":"172.21.15.93:6807/38179","heartbeat_front_addr":"172.21.15.93:6806/38179","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2022-09-08T08:24:04.259680+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":65536,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[{"pgid":"3.2","osds":[1,0,2]},{"pgid":"3.a","osds":[3,0,1]}],"pg_upmap_items":[{"pgid":"3.a","mappings":[{"from":1,"to":3}]}],"pg_temp":[{"pgid":"3.1","osds":[3,1]},{"pgid":"3.3","osds":[2,3]},{"pgid":"3.4","osds":[2,3]},{"pgid":"3.7","osds":[3,1]},{"pgid":"3.9","osds":[1,2]},{"pgid":"3.b","osds":[3,2]},{"pgid":"3.c","osds":[3,2]},{"pgid":"3.d","osds":[1,3]},{"pgid":"3.f","osds":[1,3]}],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2022-09-08T20:09:41.355 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T08:38:47.938 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~276s , 2022-09-08T08:38:49.243+0000 7fc28a12a700 -1 mon.a@0(leader) e1 get_health_metrics reporting 2 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.93:6800/38150 for 29sec e54 v54) , 2022-09-08T08:38:50.640 DEBUG:teuthology.orchestra.run.smithi085:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2022-09-08T20:09:41.356 WARNING:scrape:Ignoring malparsed backtrace: r.wait() , File "/home/teuthworker/src/git.ceph.com_git_teuthology_b58f2c18636eb10faa77ed3614abd00cb85dfc2c/teuthology/orchestra/run.py", line 161, in wait , self._raise_for_status() 2022-09-08T20:09:41.356 WARNING:scrape:Ignoring malparsed backtrace: yield vars , File "/home/teuthworker/src/github.com_Matan-B_ceph_cb98f5a0f91af829569d87fce3d044ba583a0457/qa/tasks/ceph.py", line 1902, in task , ctx.managers[config['cluster']].stop_pg_num_changes() 2022-09-08T20:09:41.357 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T08:43:23.764 INFO:tasks.ceph.osd.3:Stopped , 2022-09-08T08:43:23.765 INFO:teuthology.misc:Shutting down mgr daemons... , 2022-09-08T08:43:23.766 DEBUG:tasks.ceph.mgr.x:waiting for process to exit 2022-09-08T20:09:41.358 WARNING:scrape:Ignoring malparsed backtrace: Traceback (most recent call last): , File "/home/teuthworker/src/git.ceph.com_git_teuthology_b58f2c18636eb10faa77ed3614abd00cb85dfc2c/teuthology/contextutil.py", line 33, in nested , yield vars 2022-09-08T20:09:41.359 WARNING:scrape:Ignoring malparsed backtrace: /usr/bin/which: unrecognized option '--objects' , /usr/bin/which: unrecognized option '--max-in-flight' , /usr/bin/which: unrecognized option '--size' 2022-09-08T20:09:41.384 WARNING:scrape:Ignoring malparsed backtrace: ceph version 17.0.0-14859-ge85e1217 (e85e121747baf9691b263effef4c284a96431457) quincy (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7fa3a1a50cb0] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x229) [0x5598e2f9d04b] 2022-09-08T20:09:41.386 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T08:24:35.799 INFO:teuthology.orchestra.run.smithi085.stdout:{"epoch":51,"fsid":"11b9b9ed-354a-4cb5-853e-04e8678930c4","created":"2022-09-08T08:22:54.094684+0000","modified":"2022-09-08T08:24:15.331804+0000","last_up_change":"2022-09-08T08:24:04.259680+0000","last_in_change":"2022-09-08T08:22:55.340984+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-08T08:22:59.695319+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-09-08T08:23:03.574393+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-09-08T08:23:10.014014+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"6bf674ea-469d-492a-b3b6-94dfabcd2b5d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":24,"down_at":40,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6801","nonce":38290}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6800","nonce":38290}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6805","nonce":38290}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6804","nonce":38290}]},"public_addr":"172.21.15.85:6801/38290","cluster_addr":"172.21.15.85:6800/38290","heartbeat_back_addr":"172.21.15.85:6805/38290","heartbeat_front_addr":"172.21.15.85:6804/38290","state":["exists"]},{"osd":1,"uuid":"f496fa03-3aa3-4f1c-a182-33a22ac73734","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6803","nonce":38317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6802","nonce":38317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6806","nonce":38317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6807","nonce":38317}]},"public_addr":"172.21.15.85:6803/38317","cluster_addr":"172.21.15.85:6802/38317","heartbeat_back_addr":"172.21.15.85:6806/38317","heartbeat_front_addr":"172.21.15.85:6807/38317","state":["exists","up"]},{"osd":2,"uuid":"e78d76bc-7d8d-4f3b-8e7b-bb873df19bad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":40,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6800","nonce":38150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6801","nonce":38150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6804","nonce":38150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6805","nonce":38150}]},"public_addr":"172.21.15.93:6800/38150","cluster_addr":"172.21.15.93:6801/38150","heartbeat_back_addr":"172.21.15.93:6804/38150","heartbeat_front_addr":"172.21.15.93:6805/38150","state":["exists","up"]},{"osd":3,"uuid":"16ef1dba-c6a5-4ed2-959b-4f515867860d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6803","nonce":38179}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6802","nonce":38179}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6807","nonce":38179}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6806","nonce":38179}]},"public_addr":"172.21.15.93:6803/38179","cluster_addr":"172.21.15.93:6802/38179","heartbeat_back_addr":"172.21.15.93:6807/38179","heartbeat_front_addr":"172.21.15.93:6806/38179","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2022-09-08T08:24:04.259680+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[{"pgid":"3.2","osds":[1,0,2]},{"pgid":"3.a","osds":[3,0,1]}],"pg_upmap_items":[{"pgid":"3.a","mappings":[{"from":1,"to":3},{"from":2,"to":0}]}],"pg_temp":[{"pgid":"3.2","osds":[1,2,3]},{"pgid":"3.a","osds":[3,1,2]}],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2022-09-08T08:24:38.814 DEBUG:teuthology.orchestra.run.smithi085:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2022-09-08T08:24:39.154 INFO:teuthology.orchestra.run.smithi085.stdout: 2022-09-08T20:09:41.387 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T08:26:29.861 INFO:teuthology.orchestra.run.smithi085.stdout: , 2022-09-08T08:26:29.862 INFO:teuthology.orchestra.run.smithi085.stdout:{"epoch":51,"fsid":"11b9b9ed-354a-4cb5-853e-04e8678930c4","created":"2022-09-08T08:22:54.094684+0000","modified":"2022-09-08T08:24:15.331804+0000","last_up_change":"2022-09-08T08:24:04.259680+0000","last_in_change":"2022-09-08T08:22:55.340984+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-08T08:22:59.695319+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-09-08T08:23:03.574393+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-09-08T08:23:10.014014+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"6bf674ea-469d-492a-b3b6-94dfabcd2b5d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":24,"down_at":40,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6801","nonce":38290}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6800","nonce":38290}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6805","nonce":38290}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6804","nonce":38290}]},"public_addr":"172.21.15.85:6801/38290","cluster_addr":"172.21.15.85:6800/38290","heartbeat_back_addr":"172.21.15.85:6805/38290","heartbeat_front_addr":"172.21.15.85:6804/38290","state":["exists"]},{"osd":1,"uuid":"f496fa03-3aa3-4f1c-a182-33a22ac73734","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6803","nonce":38317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6802","nonce":38317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6806","nonce":38317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6807","nonce":38317}]},"public_addr":"172.21.15.85:6803/38317","cluster_addr":"172.21.15.85:6802/38317","heartbeat_back_addr":"172.21.15.85:6806/38317","heartbeat_front_addr":"172.21.15.85:6807/38317","state":["exists","up"]},{"osd":2,"uuid":"e78d76bc-7d8d-4f3b-8e7b-bb873df19bad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":40,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6800","nonce":38150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6801","nonce":38150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6804","nonce":38150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6805","nonce":38150}]},"public_addr":"172.21.15.93:6800/38150","cluster_addr":"172.21.15.93:6801/38150","heartbeat_back_addr":"172.21.15.93:6804/38150","heartbeat_front_addr":"172.21.15.93:6805/38150","state":["exists","up"]},{"osd":3,"uuid":"16ef1dba-c6a5-4ed2-959b-4f515867860d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6803","nonce":38179}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6802","nonce":38179}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6807","nonce":38179}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6806","nonce":38179}]},"public_addr":"172.21.15.93:6803/38179","cluster_addr":"172.21.15.93:6802/38179","heartbeat_back_addr":"172.21.15.93:6807/38179","heartbeat_front_addr":"172.21.15.93:6806/38179","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2022-09-08T08:24:04.259680+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[{"pgid":"3.2","osds":[1,0,2]},{"pgid":"3.a","osds":[3,0,1]}],"pg_upmap_items":[{"pgid":"3.a","mappings":[{"from":1,"to":3},{"from":2,"to":0}]}],"pg_temp":[{"pgid":"3.2","osds":[1,2,3]},{"pgid":"3.a","osds":[3,1,2]}],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2022-09-08T08:26:32.875 DEBUG:teuthology.orchestra.run.smithi085:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2022-09-08T20:09:41.388 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T08:28:23.507 DEBUG:teuthology.orchestra.run.smithi085:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2022-09-08T08:28:23.841 INFO:teuthology.orchestra.run.smithi085.stdout: , 2022-09-08T08:28:23.842 INFO:teuthology.orchestra.run.smithi085.stdout:{"epoch":51,"fsid":"11b9b9ed-354a-4cb5-853e-04e8678930c4","created":"2022-09-08T08:22:54.094684+0000","modified":"2022-09-08T08:24:15.331804+0000","last_up_change":"2022-09-08T08:24:04.259680+0000","last_in_change":"2022-09-08T08:22:55.340984+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-08T08:22:59.695319+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-09-08T08:23:03.574393+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-09-08T08:23:10.014014+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"6bf674ea-469d-492a-b3b6-94dfabcd2b5d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":24,"down_at":40,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6801","nonce":38290}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6800","nonce":38290}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6805","nonce":38290}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6804","nonce":38290}]},"public_addr":"172.21.15.85:6801/38290","cluster_addr":"172.21.15.85:6800/38290","heartbeat_back_addr":"172.21.15.85:6805/38290","heartbeat_front_addr":"172.21.15.85:6804/38290","state":["exists"]},{"osd":1,"uuid":"f496fa03-3aa3-4f1c-a182-33a22ac73734","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6803","nonce":38317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6802","nonce":38317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6806","nonce":38317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6807","nonce":38317}]},"public_addr":"172.21.15.85:6803/38317","cluster_addr":"172.21.15.85:6802/38317","heartbeat_back_addr":"172.21.15.85:6806/38317","heartbeat_front_addr":"172.21.15.85:6807/38317","state":["exists","up"]},{"osd":2,"uuid":"e78d76bc-7d8d-4f3b-8e7b-bb873df19bad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":40,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6800","nonce":38150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6801","nonce":38150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6804","nonce":38150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6805","nonce":38150}]},"public_addr":"172.21.15.93:6800/38150","cluster_addr":"172.21.15.93:6801/38150","heartbeat_back_addr":"172.21.15.93:6804/38150","heartbeat_front_addr":"172.21.15.93:6805/38150","state":["exists","up"]},{"osd":3,"uuid":"16ef1dba-c6a5-4ed2-959b-4f515867860d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6803","nonce":38179}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6802","nonce":38179}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6807","nonce":38179}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6806","nonce":38179}]},"public_addr":"172.21.15.93:6803/38179","cluster_addr":"172.21.15.93:6802/38179","heartbeat_back_addr":"172.21.15.93:6807/38179","heartbeat_front_addr":"172.21.15.93:6806/38179","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2022-09-08T08:24:04.259680+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[{"pgid":"3.2","osds":[1,0,2]},{"pgid":"3.a","osds":[3,0,1]}],"pg_upmap_items":[{"pgid":"3.a","mappings":[{"from":1,"to":3},{"from":2,"to":0}]}],"pg_temp":[{"pgid":"3.2","osds":[1,2,3]},{"pgid":"3.a","osds":[3,1,2]}],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2022-09-08T20:09:41.390 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T08:30:14.479 INFO:teuthology.orchestra.run.smithi085.stdout:{"epoch":51,"fsid":"11b9b9ed-354a-4cb5-853e-04e8678930c4","created":"2022-09-08T08:22:54.094684+0000","modified":"2022-09-08T08:24:15.331804+0000","last_up_change":"2022-09-08T08:24:04.259680+0000","last_in_change":"2022-09-08T08:22:55.340984+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-08T08:22:59.695319+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-09-08T08:23:03.574393+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-09-08T08:23:10.014014+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"6bf674ea-469d-492a-b3b6-94dfabcd2b5d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":24,"down_at":40,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6801","nonce":38290}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6800","nonce":38290}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6805","nonce":38290}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6804","nonce":38290}]},"public_addr":"172.21.15.85:6801/38290","cluster_addr":"172.21.15.85:6800/38290","heartbeat_back_addr":"172.21.15.85:6805/38290","heartbeat_front_addr":"172.21.15.85:6804/38290","state":["exists"]},{"osd":1,"uuid":"f496fa03-3aa3-4f1c-a182-33a22ac73734","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6803","nonce":38317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6802","nonce":38317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6806","nonce":38317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6807","nonce":38317}]},"public_addr":"172.21.15.85:6803/38317","cluster_addr":"172.21.15.85:6802/38317","heartbeat_back_addr":"172.21.15.85:6806/38317","heartbeat_front_addr":"172.21.15.85:6807/38317","state":["exists","up"]},{"osd":2,"uuid":"e78d76bc-7d8d-4f3b-8e7b-bb873df19bad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":40,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6800","nonce":38150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6801","nonce":38150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6804","nonce":38150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6805","nonce":38150}]},"public_addr":"172.21.15.93:6800/38150","cluster_addr":"172.21.15.93:6801/38150","heartbeat_back_addr":"172.21.15.93:6804/38150","heartbeat_front_addr":"172.21.15.93:6805/38150","state":["exists","up"]},{"osd":3,"uuid":"16ef1dba-c6a5-4ed2-959b-4f515867860d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6803","nonce":38179}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6802","nonce":38179}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6807","nonce":38179}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6806","nonce":38179}]},"public_addr":"172.21.15.93:6803/38179","cluster_addr":"172.21.15.93:6802/38179","heartbeat_back_addr":"172.21.15.93:6807/38179","heartbeat_front_addr":"172.21.15.93:6806/38179","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2022-09-08T08:24:04.259680+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[{"pgid":"3.2","osds":[1,0,2]},{"pgid":"3.a","osds":[3,0,1]}],"pg_upmap_items":[{"pgid":"3.a","mappings":[{"from":1,"to":3},{"from":2,"to":0}]}],"pg_temp":[{"pgid":"3.2","osds":[1,2,3]},{"pgid":"3.a","osds":[3,1,2]}],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2022-09-08T08:30:17.494 DEBUG:teuthology.orchestra.run.smithi085:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2022-09-08T08:30:17.829 INFO:teuthology.orchestra.run.smithi085.stdout: 2022-09-08T20:09:41.392 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T08:32:08.562 INFO:teuthology.orchestra.run.smithi085.stdout: , 2022-09-08T08:32:08.563 INFO:teuthology.orchestra.run.smithi085.stdout:{"epoch":51,"fsid":"11b9b9ed-354a-4cb5-853e-04e8678930c4","created":"2022-09-08T08:22:54.094684+0000","modified":"2022-09-08T08:24:15.331804+0000","last_up_change":"2022-09-08T08:24:04.259680+0000","last_in_change":"2022-09-08T08:22:55.340984+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-08T08:22:59.695319+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-09-08T08:23:03.574393+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-09-08T08:23:10.014014+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"6bf674ea-469d-492a-b3b6-94dfabcd2b5d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":24,"down_at":40,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6801","nonce":38290}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6800","nonce":38290}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6805","nonce":38290}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6804","nonce":38290}]},"public_addr":"172.21.15.85:6801/38290","cluster_addr":"172.21.15.85:6800/38290","heartbeat_back_addr":"172.21.15.85:6805/38290","heartbeat_front_addr":"172.21.15.85:6804/38290","state":["exists"]},{"osd":1,"uuid":"f496fa03-3aa3-4f1c-a182-33a22ac73734","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6803","nonce":38317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6802","nonce":38317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6806","nonce":38317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6807","nonce":38317}]},"public_addr":"172.21.15.85:6803/38317","cluster_addr":"172.21.15.85:6802/38317","heartbeat_back_addr":"172.21.15.85:6806/38317","heartbeat_front_addr":"172.21.15.85:6807/38317","state":["exists","up"]},{"osd":2,"uuid":"e78d76bc-7d8d-4f3b-8e7b-bb873df19bad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":40,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6800","nonce":38150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6801","nonce":38150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6804","nonce":38150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6805","nonce":38150}]},"public_addr":"172.21.15.93:6800/38150","cluster_addr":"172.21.15.93:6801/38150","heartbeat_back_addr":"172.21.15.93:6804/38150","heartbeat_front_addr":"172.21.15.93:6805/38150","state":["exists","up"]},{"osd":3,"uuid":"16ef1dba-c6a5-4ed2-959b-4f515867860d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6803","nonce":38179}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6802","nonce":38179}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6807","nonce":38179}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6806","nonce":38179}]},"public_addr":"172.21.15.93:6803/38179","cluster_addr":"172.21.15.93:6802/38179","heartbeat_back_addr":"172.21.15.93:6807/38179","heartbeat_front_addr":"172.21.15.93:6806/38179","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2022-09-08T08:24:04.259680+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[{"pgid":"3.2","osds":[1,0,2]},{"pgid":"3.a","osds":[3,0,1]}],"pg_upmap_items":[{"pgid":"3.a","mappings":[{"from":1,"to":3},{"from":2,"to":0}]}],"pg_temp":[{"pgid":"3.2","osds":[1,2,3]},{"pgid":"3.a","osds":[3,1,2]}],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2022-09-08T08:32:11.577 DEBUG:teuthology.orchestra.run.smithi085:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2022-09-08T20:09:41.393 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T08:34:02.231 DEBUG:teuthology.orchestra.run.smithi085:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2022-09-08T08:34:02.568 INFO:teuthology.orchestra.run.smithi085.stdout: , 2022-09-08T08:34:02.568 INFO:teuthology.orchestra.run.smithi085.stdout:{"epoch":51,"fsid":"11b9b9ed-354a-4cb5-853e-04e8678930c4","created":"2022-09-08T08:22:54.094684+0000","modified":"2022-09-08T08:24:15.331804+0000","last_up_change":"2022-09-08T08:24:04.259680+0000","last_in_change":"2022-09-08T08:22:55.340984+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-08T08:22:59.695319+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-09-08T08:23:03.574393+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-09-08T08:23:10.014014+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"6bf674ea-469d-492a-b3b6-94dfabcd2b5d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":24,"down_at":40,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6801","nonce":38290}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6800","nonce":38290}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6805","nonce":38290}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6804","nonce":38290}]},"public_addr":"172.21.15.85:6801/38290","cluster_addr":"172.21.15.85:6800/38290","heartbeat_back_addr":"172.21.15.85:6805/38290","heartbeat_front_addr":"172.21.15.85:6804/38290","state":["exists"]},{"osd":1,"uuid":"f496fa03-3aa3-4f1c-a182-33a22ac73734","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6803","nonce":38317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6802","nonce":38317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6806","nonce":38317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6807","nonce":38317}]},"public_addr":"172.21.15.85:6803/38317","cluster_addr":"172.21.15.85:6802/38317","heartbeat_back_addr":"172.21.15.85:6806/38317","heartbeat_front_addr":"172.21.15.85:6807/38317","state":["exists","up"]},{"osd":2,"uuid":"e78d76bc-7d8d-4f3b-8e7b-bb873df19bad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":40,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6800","nonce":38150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6801","nonce":38150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6804","nonce":38150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6805","nonce":38150}]},"public_addr":"172.21.15.93:6800/38150","cluster_addr":"172.21.15.93:6801/38150","heartbeat_back_addr":"172.21.15.93:6804/38150","heartbeat_front_addr":"172.21.15.93:6805/38150","state":["exists","up"]},{"osd":3,"uuid":"16ef1dba-c6a5-4ed2-959b-4f515867860d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6803","nonce":38179}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6802","nonce":38179}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6807","nonce":38179}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6806","nonce":38179}]},"public_addr":"172.21.15.93:6803/38179","cluster_addr":"172.21.15.93:6802/38179","heartbeat_back_addr":"172.21.15.93:6807/38179","heartbeat_front_addr":"172.21.15.93:6806/38179","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2022-09-08T08:24:04.259680+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[{"pgid":"3.2","osds":[1,0,2]},{"pgid":"3.a","osds":[3,0,1]}],"pg_upmap_items":[{"pgid":"3.a","mappings":[{"from":1,"to":3},{"from":2,"to":0}]}],"pg_temp":[{"pgid":"3.2","osds":[1,2,3]},{"pgid":"3.a","osds":[3,1,2]}],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2022-09-08T20:09:41.394 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T08:35:22.554 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~70s , 2022-09-08T08:35:22.735 DEBUG:teuthology.orchestra.run.smithi085:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2022-09-08T08:35:23.075 INFO:teuthology.orchestra.run.smithi085.stdout: 2022-09-08T20:09:41.395 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T08:36:30.191 INFO:teuthology.orchestra.run.smithi085.stdout:{"epoch":54,"fsid":"11b9b9ed-354a-4cb5-853e-04e8678930c4","created":"2022-09-08T08:22:54.094684+0000","modified":"2022-09-08T08:34:11.194702+0000","last_up_change":"2022-09-08T08:24:04.259680+0000","last_in_change":"2022-09-08T08:34:09.186829+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-08T08:22:59.695319+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-09-08T08:23:03.574393+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-09-08T08:23:10.014014+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"6bf674ea-469d-492a-b3b6-94dfabcd2b5d","up":0,"in":0,"weight":0,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":24,"down_at":40,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6801","nonce":38290}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6800","nonce":38290}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6805","nonce":38290}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6804","nonce":38290}]},"public_addr":"172.21.15.85:6801/38290","cluster_addr":"172.21.15.85:6800/38290","heartbeat_back_addr":"172.21.15.85:6805/38290","heartbeat_front_addr":"172.21.15.85:6804/38290","state":["autoout","exists"]},{"osd":1,"uuid":"f496fa03-3aa3-4f1c-a182-33a22ac73734","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6803","nonce":38317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6802","nonce":38317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6806","nonce":38317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6807","nonce":38317}]},"public_addr":"172.21.15.85:6803/38317","cluster_addr":"172.21.15.85:6802/38317","heartbeat_back_addr":"172.21.15.85:6806/38317","heartbeat_front_addr":"172.21.15.85:6807/38317","state":["exists","up"]},{"osd":2,"uuid":"e78d76bc-7d8d-4f3b-8e7b-bb873df19bad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6800","nonce":38150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6801","nonce":38150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6804","nonce":38150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6805","nonce":38150}]},"public_addr":"172.21.15.93:6800/38150","cluster_addr":"172.21.15.93:6801/38150","heartbeat_back_addr":"172.21.15.93:6804/38150","heartbeat_front_addr":"172.21.15.93:6805/38150","state":["exists","up"]},{"osd":3,"uuid":"16ef1dba-c6a5-4ed2-959b-4f515867860d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6803","nonce":38179}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6802","nonce":38179}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6807","nonce":38179}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6806","nonce":38179}]},"public_addr":"172.21.15.93:6803/38179","cluster_addr":"172.21.15.93:6802/38179","heartbeat_back_addr":"172.21.15.93:6807/38179","heartbeat_front_addr":"172.21.15.93:6806/38179","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2022-09-08T08:24:04.259680+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":65536,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[{"pgid":"3.2","osds":[1,0,2]},{"pgid":"3.a","osds":[3,0,1]}],"pg_upmap_items":[{"pgid":"3.a","mappings":[{"from":1,"to":3}]}],"pg_temp":[{"pgid":"3.1","osds":[3,1]},{"pgid":"3.3","osds":[2,3]},{"pgid":"3.4","osds":[2,3]},{"pgid":"3.7","osds":[3,1]},{"pgid":"3.9","osds":[1,2]},{"pgid":"3.b","osds":[3,2]},{"pgid":"3.c","osds":[3,2]},{"pgid":"3.d","osds":[1,3]},{"pgid":"3.f","osds":[1,3]}],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2022-09-08T08:36:32.816 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.2 is failed for ~141s , 2022-09-08T08:36:32.816 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~141s 2022-09-08T20:09:41.396 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T08:37:40.242 DEBUG:teuthology.orchestra.run.smithi085:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2022-09-08T08:37:40.583 INFO:teuthology.orchestra.run.smithi085.stdout: , 2022-09-08T08:37:40.583 INFO:teuthology.orchestra.run.smithi085.stdout:{"epoch":54,"fsid":"11b9b9ed-354a-4cb5-853e-04e8678930c4","created":"2022-09-08T08:22:54.094684+0000","modified":"2022-09-08T08:34:11.194702+0000","last_up_change":"2022-09-08T08:24:04.259680+0000","last_in_change":"2022-09-08T08:34:09.186829+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-08T08:22:59.695319+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-09-08T08:23:03.574393+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-09-08T08:23:10.014014+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"6bf674ea-469d-492a-b3b6-94dfabcd2b5d","up":0,"in":0,"weight":0,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":24,"down_at":40,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6801","nonce":38290}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6800","nonce":38290}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6805","nonce":38290}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6804","nonce":38290}]},"public_addr":"172.21.15.85:6801/38290","cluster_addr":"172.21.15.85:6800/38290","heartbeat_back_addr":"172.21.15.85:6805/38290","heartbeat_front_addr":"172.21.15.85:6804/38290","state":["autoout","exists"]},{"osd":1,"uuid":"f496fa03-3aa3-4f1c-a182-33a22ac73734","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6803","nonce":38317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6802","nonce":38317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6806","nonce":38317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6807","nonce":38317}]},"public_addr":"172.21.15.85:6803/38317","cluster_addr":"172.21.15.85:6802/38317","heartbeat_back_addr":"172.21.15.85:6806/38317","heartbeat_front_addr":"172.21.15.85:6807/38317","state":["exists","up"]},{"osd":2,"uuid":"e78d76bc-7d8d-4f3b-8e7b-bb873df19bad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6800","nonce":38150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6801","nonce":38150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6804","nonce":38150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6805","nonce":38150}]},"public_addr":"172.21.15.93:6800/38150","cluster_addr":"172.21.15.93:6801/38150","heartbeat_back_addr":"172.21.15.93:6804/38150","heartbeat_front_addr":"172.21.15.93:6805/38150","state":["exists","up"]},{"osd":3,"uuid":"16ef1dba-c6a5-4ed2-959b-4f515867860d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6803","nonce":38179}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6802","nonce":38179}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6807","nonce":38179}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6806","nonce":38179}]},"public_addr":"172.21.15.93:6803/38179","cluster_addr":"172.21.15.93:6802/38179","heartbeat_back_addr":"172.21.15.93:6807/38179","heartbeat_front_addr":"172.21.15.93:6806/38179","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2022-09-08T08:24:04.259680+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":65536,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[{"pgid":"3.2","osds":[1,0,2]},{"pgid":"3.a","osds":[3,0,1]}],"pg_upmap_items":[{"pgid":"3.a","mappings":[{"from":1,"to":3}]}],"pg_temp":[{"pgid":"3.1","osds":[3,1]},{"pgid":"3.3","osds":[2,3]},{"pgid":"3.4","osds":[2,3]},{"pgid":"3.7","osds":[3,1]},{"pgid":"3.9","osds":[1,2]},{"pgid":"3.b","osds":[3,2]},{"pgid":"3.c","osds":[3,2]},{"pgid":"3.d","osds":[1,3]},{"pgid":"3.f","osds":[1,3]}],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2022-09-08T20:09:41.397 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T08:38:47.938 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~276s , 2022-09-08T08:38:49.243+0000 7fc28a12a700 -1 mon.a@0(leader) e1 get_health_metrics reporting 2 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.93:6800/38150 for 29sec e54 v54) , 2022-09-08T08:38:50.640 DEBUG:teuthology.orchestra.run.smithi085:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2022-09-08T20:09:41.398 WARNING:scrape:Ignoring malparsed backtrace: r.wait() , File "/home/teuthworker/src/git.ceph.com_git_teuthology_b58f2c18636eb10faa77ed3614abd00cb85dfc2c/teuthology/orchestra/run.py", line 161, in wait , self._raise_for_status() 2022-09-08T20:09:41.399 WARNING:scrape:Ignoring malparsed backtrace: yield vars , File "/home/teuthworker/src/github.com_Matan-B_ceph_cb98f5a0f91af829569d87fce3d044ba583a0457/qa/tasks/ceph.py", line 1902, in task , ctx.managers[config['cluster']].stop_pg_num_changes() 2022-09-08T20:09:41.400 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T08:43:23.764 INFO:tasks.ceph.osd.3:Stopped , 2022-09-08T08:43:23.765 INFO:teuthology.misc:Shutting down mgr daemons... , 2022-09-08T08:43:23.766 DEBUG:tasks.ceph.mgr.x:waiting for process to exit 2022-09-08T20:09:41.400 WARNING:scrape:Ignoring malparsed backtrace: Traceback (most recent call last): , File "/home/teuthworker/src/git.ceph.com_git_teuthology_b58f2c18636eb10faa77ed3614abd00cb85dfc2c/teuthology/contextutil.py", line 33, in nested , yield vars 2022-09-08T20:09:41.401 WARNING:scrape:Ignoring malparsed backtrace: /usr/bin/which: unrecognized option '--objects' , /usr/bin/which: unrecognized option '--max-in-flight' , /usr/bin/which: unrecognized option '--size' 2022-09-08T20:09:41.432 WARNING:scrape:Ignoring malparsed backtrace: ceph version 17.0.0-14859-ge85e1217 (e85e121747baf9691b263effef4c284a96431457) quincy (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7fa3a1a50cb0] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x229) [0x5598e2f9d04b] 2022-09-08T20:09:41.434 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T08:24:35.799 INFO:teuthology.orchestra.run.smithi085.stdout:{"epoch":51,"fsid":"11b9b9ed-354a-4cb5-853e-04e8678930c4","created":"2022-09-08T08:22:54.094684+0000","modified":"2022-09-08T08:24:15.331804+0000","last_up_change":"2022-09-08T08:24:04.259680+0000","last_in_change":"2022-09-08T08:22:55.340984+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-08T08:22:59.695319+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-09-08T08:23:03.574393+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-09-08T08:23:10.014014+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"6bf674ea-469d-492a-b3b6-94dfabcd2b5d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":24,"down_at":40,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6801","nonce":38290}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6800","nonce":38290}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6805","nonce":38290}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6804","nonce":38290}]},"public_addr":"172.21.15.85:6801/38290","cluster_addr":"172.21.15.85:6800/38290","heartbeat_back_addr":"172.21.15.85:6805/38290","heartbeat_front_addr":"172.21.15.85:6804/38290","state":["exists"]},{"osd":1,"uuid":"f496fa03-3aa3-4f1c-a182-33a22ac73734","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6803","nonce":38317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6802","nonce":38317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6806","nonce":38317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6807","nonce":38317}]},"public_addr":"172.21.15.85:6803/38317","cluster_addr":"172.21.15.85:6802/38317","heartbeat_back_addr":"172.21.15.85:6806/38317","heartbeat_front_addr":"172.21.15.85:6807/38317","state":["exists","up"]},{"osd":2,"uuid":"e78d76bc-7d8d-4f3b-8e7b-bb873df19bad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":40,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6800","nonce":38150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6801","nonce":38150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6804","nonce":38150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6805","nonce":38150}]},"public_addr":"172.21.15.93:6800/38150","cluster_addr":"172.21.15.93:6801/38150","heartbeat_back_addr":"172.21.15.93:6804/38150","heartbeat_front_addr":"172.21.15.93:6805/38150","state":["exists","up"]},{"osd":3,"uuid":"16ef1dba-c6a5-4ed2-959b-4f515867860d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6803","nonce":38179}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6802","nonce":38179}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6807","nonce":38179}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6806","nonce":38179}]},"public_addr":"172.21.15.93:6803/38179","cluster_addr":"172.21.15.93:6802/38179","heartbeat_back_addr":"172.21.15.93:6807/38179","heartbeat_front_addr":"172.21.15.93:6806/38179","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2022-09-08T08:24:04.259680+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[{"pgid":"3.2","osds":[1,0,2]},{"pgid":"3.a","osds":[3,0,1]}],"pg_upmap_items":[{"pgid":"3.a","mappings":[{"from":1,"to":3},{"from":2,"to":0}]}],"pg_temp":[{"pgid":"3.2","osds":[1,2,3]},{"pgid":"3.a","osds":[3,1,2]}],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2022-09-08T08:24:38.814 DEBUG:teuthology.orchestra.run.smithi085:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2022-09-08T08:24:39.154 INFO:teuthology.orchestra.run.smithi085.stdout: 2022-09-08T20:09:41.437 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T08:26:29.861 INFO:teuthology.orchestra.run.smithi085.stdout: , 2022-09-08T08:26:29.862 INFO:teuthology.orchestra.run.smithi085.stdout:{"epoch":51,"fsid":"11b9b9ed-354a-4cb5-853e-04e8678930c4","created":"2022-09-08T08:22:54.094684+0000","modified":"2022-09-08T08:24:15.331804+0000","last_up_change":"2022-09-08T08:24:04.259680+0000","last_in_change":"2022-09-08T08:22:55.340984+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-08T08:22:59.695319+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-09-08T08:23:03.574393+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-09-08T08:23:10.014014+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"6bf674ea-469d-492a-b3b6-94dfabcd2b5d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":24,"down_at":40,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6801","nonce":38290}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6800","nonce":38290}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6805","nonce":38290}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6804","nonce":38290}]},"public_addr":"172.21.15.85:6801/38290","cluster_addr":"172.21.15.85:6800/38290","heartbeat_back_addr":"172.21.15.85:6805/38290","heartbeat_front_addr":"172.21.15.85:6804/38290","state":["exists"]},{"osd":1,"uuid":"f496fa03-3aa3-4f1c-a182-33a22ac73734","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6803","nonce":38317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6802","nonce":38317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6806","nonce":38317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6807","nonce":38317}]},"public_addr":"172.21.15.85:6803/38317","cluster_addr":"172.21.15.85:6802/38317","heartbeat_back_addr":"172.21.15.85:6806/38317","heartbeat_front_addr":"172.21.15.85:6807/38317","state":["exists","up"]},{"osd":2,"uuid":"e78d76bc-7d8d-4f3b-8e7b-bb873df19bad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":40,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6800","nonce":38150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6801","nonce":38150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6804","nonce":38150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6805","nonce":38150}]},"public_addr":"172.21.15.93:6800/38150","cluster_addr":"172.21.15.93:6801/38150","heartbeat_back_addr":"172.21.15.93:6804/38150","heartbeat_front_addr":"172.21.15.93:6805/38150","state":["exists","up"]},{"osd":3,"uuid":"16ef1dba-c6a5-4ed2-959b-4f515867860d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6803","nonce":38179}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6802","nonce":38179}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6807","nonce":38179}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6806","nonce":38179}]},"public_addr":"172.21.15.93:6803/38179","cluster_addr":"172.21.15.93:6802/38179","heartbeat_back_addr":"172.21.15.93:6807/38179","heartbeat_front_addr":"172.21.15.93:6806/38179","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2022-09-08T08:24:04.259680+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[{"pgid":"3.2","osds":[1,0,2]},{"pgid":"3.a","osds":[3,0,1]}],"pg_upmap_items":[{"pgid":"3.a","mappings":[{"from":1,"to":3},{"from":2,"to":0}]}],"pg_temp":[{"pgid":"3.2","osds":[1,2,3]},{"pgid":"3.a","osds":[3,1,2]}],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2022-09-08T08:26:32.875 DEBUG:teuthology.orchestra.run.smithi085:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2022-09-08T20:09:41.439 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T08:28:23.507 DEBUG:teuthology.orchestra.run.smithi085:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2022-09-08T08:28:23.841 INFO:teuthology.orchestra.run.smithi085.stdout: , 2022-09-08T08:28:23.842 INFO:teuthology.orchestra.run.smithi085.stdout:{"epoch":51,"fsid":"11b9b9ed-354a-4cb5-853e-04e8678930c4","created":"2022-09-08T08:22:54.094684+0000","modified":"2022-09-08T08:24:15.331804+0000","last_up_change":"2022-09-08T08:24:04.259680+0000","last_in_change":"2022-09-08T08:22:55.340984+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-08T08:22:59.695319+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-09-08T08:23:03.574393+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-09-08T08:23:10.014014+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"6bf674ea-469d-492a-b3b6-94dfabcd2b5d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":24,"down_at":40,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6801","nonce":38290}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6800","nonce":38290}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6805","nonce":38290}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6804","nonce":38290}]},"public_addr":"172.21.15.85:6801/38290","cluster_addr":"172.21.15.85:6800/38290","heartbeat_back_addr":"172.21.15.85:6805/38290","heartbeat_front_addr":"172.21.15.85:6804/38290","state":["exists"]},{"osd":1,"uuid":"f496fa03-3aa3-4f1c-a182-33a22ac73734","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6803","nonce":38317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6802","nonce":38317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6806","nonce":38317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6807","nonce":38317}]},"public_addr":"172.21.15.85:6803/38317","cluster_addr":"172.21.15.85:6802/38317","heartbeat_back_addr":"172.21.15.85:6806/38317","heartbeat_front_addr":"172.21.15.85:6807/38317","state":["exists","up"]},{"osd":2,"uuid":"e78d76bc-7d8d-4f3b-8e7b-bb873df19bad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":40,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6800","nonce":38150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6801","nonce":38150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6804","nonce":38150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6805","nonce":38150}]},"public_addr":"172.21.15.93:6800/38150","cluster_addr":"172.21.15.93:6801/38150","heartbeat_back_addr":"172.21.15.93:6804/38150","heartbeat_front_addr":"172.21.15.93:6805/38150","state":["exists","up"]},{"osd":3,"uuid":"16ef1dba-c6a5-4ed2-959b-4f515867860d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6803","nonce":38179}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6802","nonce":38179}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6807","nonce":38179}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6806","nonce":38179}]},"public_addr":"172.21.15.93:6803/38179","cluster_addr":"172.21.15.93:6802/38179","heartbeat_back_addr":"172.21.15.93:6807/38179","heartbeat_front_addr":"172.21.15.93:6806/38179","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2022-09-08T08:24:04.259680+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[{"pgid":"3.2","osds":[1,0,2]},{"pgid":"3.a","osds":[3,0,1]}],"pg_upmap_items":[{"pgid":"3.a","mappings":[{"from":1,"to":3},{"from":2,"to":0}]}],"pg_temp":[{"pgid":"3.2","osds":[1,2,3]},{"pgid":"3.a","osds":[3,1,2]}],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2022-09-08T20:09:41.441 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T08:30:14.479 INFO:teuthology.orchestra.run.smithi085.stdout:{"epoch":51,"fsid":"11b9b9ed-354a-4cb5-853e-04e8678930c4","created":"2022-09-08T08:22:54.094684+0000","modified":"2022-09-08T08:24:15.331804+0000","last_up_change":"2022-09-08T08:24:04.259680+0000","last_in_change":"2022-09-08T08:22:55.340984+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-08T08:22:59.695319+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-09-08T08:23:03.574393+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-09-08T08:23:10.014014+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"6bf674ea-469d-492a-b3b6-94dfabcd2b5d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":24,"down_at":40,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6801","nonce":38290}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6800","nonce":38290}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6805","nonce":38290}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6804","nonce":38290}]},"public_addr":"172.21.15.85:6801/38290","cluster_addr":"172.21.15.85:6800/38290","heartbeat_back_addr":"172.21.15.85:6805/38290","heartbeat_front_addr":"172.21.15.85:6804/38290","state":["exists"]},{"osd":1,"uuid":"f496fa03-3aa3-4f1c-a182-33a22ac73734","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6803","nonce":38317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6802","nonce":38317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6806","nonce":38317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6807","nonce":38317}]},"public_addr":"172.21.15.85:6803/38317","cluster_addr":"172.21.15.85:6802/38317","heartbeat_back_addr":"172.21.15.85:6806/38317","heartbeat_front_addr":"172.21.15.85:6807/38317","state":["exists","up"]},{"osd":2,"uuid":"e78d76bc-7d8d-4f3b-8e7b-bb873df19bad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":40,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6800","nonce":38150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6801","nonce":38150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6804","nonce":38150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6805","nonce":38150}]},"public_addr":"172.21.15.93:6800/38150","cluster_addr":"172.21.15.93:6801/38150","heartbeat_back_addr":"172.21.15.93:6804/38150","heartbeat_front_addr":"172.21.15.93:6805/38150","state":["exists","up"]},{"osd":3,"uuid":"16ef1dba-c6a5-4ed2-959b-4f515867860d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6803","nonce":38179}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6802","nonce":38179}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6807","nonce":38179}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6806","nonce":38179}]},"public_addr":"172.21.15.93:6803/38179","cluster_addr":"172.21.15.93:6802/38179","heartbeat_back_addr":"172.21.15.93:6807/38179","heartbeat_front_addr":"172.21.15.93:6806/38179","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2022-09-08T08:24:04.259680+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[{"pgid":"3.2","osds":[1,0,2]},{"pgid":"3.a","osds":[3,0,1]}],"pg_upmap_items":[{"pgid":"3.a","mappings":[{"from":1,"to":3},{"from":2,"to":0}]}],"pg_temp":[{"pgid":"3.2","osds":[1,2,3]},{"pgid":"3.a","osds":[3,1,2]}],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2022-09-08T08:30:17.494 DEBUG:teuthology.orchestra.run.smithi085:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2022-09-08T08:30:17.829 INFO:teuthology.orchestra.run.smithi085.stdout: 2022-09-08T20:09:41.443 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T08:32:08.562 INFO:teuthology.orchestra.run.smithi085.stdout: , 2022-09-08T08:32:08.563 INFO:teuthology.orchestra.run.smithi085.stdout:{"epoch":51,"fsid":"11b9b9ed-354a-4cb5-853e-04e8678930c4","created":"2022-09-08T08:22:54.094684+0000","modified":"2022-09-08T08:24:15.331804+0000","last_up_change":"2022-09-08T08:24:04.259680+0000","last_in_change":"2022-09-08T08:22:55.340984+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-08T08:22:59.695319+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-09-08T08:23:03.574393+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-09-08T08:23:10.014014+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"6bf674ea-469d-492a-b3b6-94dfabcd2b5d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":24,"down_at":40,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6801","nonce":38290}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6800","nonce":38290}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6805","nonce":38290}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6804","nonce":38290}]},"public_addr":"172.21.15.85:6801/38290","cluster_addr":"172.21.15.85:6800/38290","heartbeat_back_addr":"172.21.15.85:6805/38290","heartbeat_front_addr":"172.21.15.85:6804/38290","state":["exists"]},{"osd":1,"uuid":"f496fa03-3aa3-4f1c-a182-33a22ac73734","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6803","nonce":38317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6802","nonce":38317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6806","nonce":38317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6807","nonce":38317}]},"public_addr":"172.21.15.85:6803/38317","cluster_addr":"172.21.15.85:6802/38317","heartbeat_back_addr":"172.21.15.85:6806/38317","heartbeat_front_addr":"172.21.15.85:6807/38317","state":["exists","up"]},{"osd":2,"uuid":"e78d76bc-7d8d-4f3b-8e7b-bb873df19bad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":40,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6800","nonce":38150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6801","nonce":38150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6804","nonce":38150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6805","nonce":38150}]},"public_addr":"172.21.15.93:6800/38150","cluster_addr":"172.21.15.93:6801/38150","heartbeat_back_addr":"172.21.15.93:6804/38150","heartbeat_front_addr":"172.21.15.93:6805/38150","state":["exists","up"]},{"osd":3,"uuid":"16ef1dba-c6a5-4ed2-959b-4f515867860d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6803","nonce":38179}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6802","nonce":38179}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6807","nonce":38179}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6806","nonce":38179}]},"public_addr":"172.21.15.93:6803/38179","cluster_addr":"172.21.15.93:6802/38179","heartbeat_back_addr":"172.21.15.93:6807/38179","heartbeat_front_addr":"172.21.15.93:6806/38179","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2022-09-08T08:24:04.259680+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[{"pgid":"3.2","osds":[1,0,2]},{"pgid":"3.a","osds":[3,0,1]}],"pg_upmap_items":[{"pgid":"3.a","mappings":[{"from":1,"to":3},{"from":2,"to":0}]}],"pg_temp":[{"pgid":"3.2","osds":[1,2,3]},{"pgid":"3.a","osds":[3,1,2]}],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2022-09-08T08:32:11.577 DEBUG:teuthology.orchestra.run.smithi085:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2022-09-08T20:09:41.445 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T08:34:02.231 DEBUG:teuthology.orchestra.run.smithi085:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2022-09-08T08:34:02.568 INFO:teuthology.orchestra.run.smithi085.stdout: , 2022-09-08T08:34:02.568 INFO:teuthology.orchestra.run.smithi085.stdout:{"epoch":51,"fsid":"11b9b9ed-354a-4cb5-853e-04e8678930c4","created":"2022-09-08T08:22:54.094684+0000","modified":"2022-09-08T08:24:15.331804+0000","last_up_change":"2022-09-08T08:24:04.259680+0000","last_in_change":"2022-09-08T08:22:55.340984+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-08T08:22:59.695319+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-09-08T08:23:03.574393+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-09-08T08:23:10.014014+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"6bf674ea-469d-492a-b3b6-94dfabcd2b5d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":24,"down_at":40,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6801","nonce":38290}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6800","nonce":38290}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6805","nonce":38290}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6804","nonce":38290}]},"public_addr":"172.21.15.85:6801/38290","cluster_addr":"172.21.15.85:6800/38290","heartbeat_back_addr":"172.21.15.85:6805/38290","heartbeat_front_addr":"172.21.15.85:6804/38290","state":["exists"]},{"osd":1,"uuid":"f496fa03-3aa3-4f1c-a182-33a22ac73734","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6803","nonce":38317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6802","nonce":38317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6806","nonce":38317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6807","nonce":38317}]},"public_addr":"172.21.15.85:6803/38317","cluster_addr":"172.21.15.85:6802/38317","heartbeat_back_addr":"172.21.15.85:6806/38317","heartbeat_front_addr":"172.21.15.85:6807/38317","state":["exists","up"]},{"osd":2,"uuid":"e78d76bc-7d8d-4f3b-8e7b-bb873df19bad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":40,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6800","nonce":38150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6801","nonce":38150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6804","nonce":38150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6805","nonce":38150}]},"public_addr":"172.21.15.93:6800/38150","cluster_addr":"172.21.15.93:6801/38150","heartbeat_back_addr":"172.21.15.93:6804/38150","heartbeat_front_addr":"172.21.15.93:6805/38150","state":["exists","up"]},{"osd":3,"uuid":"16ef1dba-c6a5-4ed2-959b-4f515867860d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6803","nonce":38179}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6802","nonce":38179}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6807","nonce":38179}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6806","nonce":38179}]},"public_addr":"172.21.15.93:6803/38179","cluster_addr":"172.21.15.93:6802/38179","heartbeat_back_addr":"172.21.15.93:6807/38179","heartbeat_front_addr":"172.21.15.93:6806/38179","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2022-09-08T08:24:04.259680+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[{"pgid":"3.2","osds":[1,0,2]},{"pgid":"3.a","osds":[3,0,1]}],"pg_upmap_items":[{"pgid":"3.a","mappings":[{"from":1,"to":3},{"from":2,"to":0}]}],"pg_temp":[{"pgid":"3.2","osds":[1,2,3]},{"pgid":"3.a","osds":[3,1,2]}],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2022-09-08T20:09:41.447 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T08:35:22.554 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~70s , 2022-09-08T08:35:22.735 DEBUG:teuthology.orchestra.run.smithi085:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2022-09-08T08:35:23.075 INFO:teuthology.orchestra.run.smithi085.stdout: 2022-09-08T20:09:41.449 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T08:36:30.191 INFO:teuthology.orchestra.run.smithi085.stdout:{"epoch":54,"fsid":"11b9b9ed-354a-4cb5-853e-04e8678930c4","created":"2022-09-08T08:22:54.094684+0000","modified":"2022-09-08T08:34:11.194702+0000","last_up_change":"2022-09-08T08:24:04.259680+0000","last_in_change":"2022-09-08T08:34:09.186829+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-08T08:22:59.695319+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-09-08T08:23:03.574393+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-09-08T08:23:10.014014+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"6bf674ea-469d-492a-b3b6-94dfabcd2b5d","up":0,"in":0,"weight":0,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":24,"down_at":40,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6801","nonce":38290}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6800","nonce":38290}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6805","nonce":38290}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6804","nonce":38290}]},"public_addr":"172.21.15.85:6801/38290","cluster_addr":"172.21.15.85:6800/38290","heartbeat_back_addr":"172.21.15.85:6805/38290","heartbeat_front_addr":"172.21.15.85:6804/38290","state":["autoout","exists"]},{"osd":1,"uuid":"f496fa03-3aa3-4f1c-a182-33a22ac73734","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6803","nonce":38317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6802","nonce":38317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6806","nonce":38317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6807","nonce":38317}]},"public_addr":"172.21.15.85:6803/38317","cluster_addr":"172.21.15.85:6802/38317","heartbeat_back_addr":"172.21.15.85:6806/38317","heartbeat_front_addr":"172.21.15.85:6807/38317","state":["exists","up"]},{"osd":2,"uuid":"e78d76bc-7d8d-4f3b-8e7b-bb873df19bad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6800","nonce":38150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6801","nonce":38150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6804","nonce":38150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6805","nonce":38150}]},"public_addr":"172.21.15.93:6800/38150","cluster_addr":"172.21.15.93:6801/38150","heartbeat_back_addr":"172.21.15.93:6804/38150","heartbeat_front_addr":"172.21.15.93:6805/38150","state":["exists","up"]},{"osd":3,"uuid":"16ef1dba-c6a5-4ed2-959b-4f515867860d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6803","nonce":38179}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6802","nonce":38179}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6807","nonce":38179}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6806","nonce":38179}]},"public_addr":"172.21.15.93:6803/38179","cluster_addr":"172.21.15.93:6802/38179","heartbeat_back_addr":"172.21.15.93:6807/38179","heartbeat_front_addr":"172.21.15.93:6806/38179","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2022-09-08T08:24:04.259680+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":65536,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[{"pgid":"3.2","osds":[1,0,2]},{"pgid":"3.a","osds":[3,0,1]}],"pg_upmap_items":[{"pgid":"3.a","mappings":[{"from":1,"to":3}]}],"pg_temp":[{"pgid":"3.1","osds":[3,1]},{"pgid":"3.3","osds":[2,3]},{"pgid":"3.4","osds":[2,3]},{"pgid":"3.7","osds":[3,1]},{"pgid":"3.9","osds":[1,2]},{"pgid":"3.b","osds":[3,2]},{"pgid":"3.c","osds":[3,2]},{"pgid":"3.d","osds":[1,3]},{"pgid":"3.f","osds":[1,3]}],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2022-09-08T08:36:32.816 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.2 is failed for ~141s , 2022-09-08T08:36:32.816 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~141s 2022-09-08T20:09:41.451 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T08:37:40.242 DEBUG:teuthology.orchestra.run.smithi085:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2022-09-08T08:37:40.583 INFO:teuthology.orchestra.run.smithi085.stdout: , 2022-09-08T08:37:40.583 INFO:teuthology.orchestra.run.smithi085.stdout:{"epoch":54,"fsid":"11b9b9ed-354a-4cb5-853e-04e8678930c4","created":"2022-09-08T08:22:54.094684+0000","modified":"2022-09-08T08:34:11.194702+0000","last_up_change":"2022-09-08T08:24:04.259680+0000","last_in_change":"2022-09-08T08:34:09.186829+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-08T08:22:59.695319+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-09-08T08:23:03.574393+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-09-08T08:23:10.014014+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"6bf674ea-469d-492a-b3b6-94dfabcd2b5d","up":0,"in":0,"weight":0,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":24,"down_at":40,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6801","nonce":38290}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6800","nonce":38290}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6805","nonce":38290}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6804","nonce":38290}]},"public_addr":"172.21.15.85:6801/38290","cluster_addr":"172.21.15.85:6800/38290","heartbeat_back_addr":"172.21.15.85:6805/38290","heartbeat_front_addr":"172.21.15.85:6804/38290","state":["autoout","exists"]},{"osd":1,"uuid":"f496fa03-3aa3-4f1c-a182-33a22ac73734","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6803","nonce":38317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6802","nonce":38317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6806","nonce":38317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.85:6807","nonce":38317}]},"public_addr":"172.21.15.85:6803/38317","cluster_addr":"172.21.15.85:6802/38317","heartbeat_back_addr":"172.21.15.85:6806/38317","heartbeat_front_addr":"172.21.15.85:6807/38317","state":["exists","up"]},{"osd":2,"uuid":"e78d76bc-7d8d-4f3b-8e7b-bb873df19bad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6800","nonce":38150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6801","nonce":38150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6804","nonce":38150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6805","nonce":38150}]},"public_addr":"172.21.15.93:6800/38150","cluster_addr":"172.21.15.93:6801/38150","heartbeat_back_addr":"172.21.15.93:6804/38150","heartbeat_front_addr":"172.21.15.93:6805/38150","state":["exists","up"]},{"osd":3,"uuid":"16ef1dba-c6a5-4ed2-959b-4f515867860d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6803","nonce":38179}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6802","nonce":38179}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6807","nonce":38179}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6806","nonce":38179}]},"public_addr":"172.21.15.93:6803/38179","cluster_addr":"172.21.15.93:6802/38179","heartbeat_back_addr":"172.21.15.93:6807/38179","heartbeat_front_addr":"172.21.15.93:6806/38179","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2022-09-08T08:24:04.259680+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":65536,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[{"pgid":"3.2","osds":[1,0,2]},{"pgid":"3.a","osds":[3,0,1]}],"pg_upmap_items":[{"pgid":"3.a","mappings":[{"from":1,"to":3}]}],"pg_temp":[{"pgid":"3.1","osds":[3,1]},{"pgid":"3.3","osds":[2,3]},{"pgid":"3.4","osds":[2,3]},{"pgid":"3.7","osds":[3,1]},{"pgid":"3.9","osds":[1,2]},{"pgid":"3.b","osds":[3,2]},{"pgid":"3.c","osds":[3,2]},{"pgid":"3.d","osds":[1,3]},{"pgid":"3.f","osds":[1,3]}],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2022-09-08T20:09:41.453 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T08:38:47.938 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~276s , 2022-09-08T08:38:49.243+0000 7fc28a12a700 -1 mon.a@0(leader) e1 get_health_metrics reporting 2 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.93:6800/38150 for 29sec e54 v54) , 2022-09-08T08:38:50.640 DEBUG:teuthology.orchestra.run.smithi085:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2022-09-08T20:09:41.454 WARNING:scrape:Ignoring malparsed backtrace: r.wait() , File "/home/teuthworker/src/git.ceph.com_git_teuthology_b58f2c18636eb10faa77ed3614abd00cb85dfc2c/teuthology/orchestra/run.py", line 161, in wait , self._raise_for_status() 2022-09-08T20:09:41.455 WARNING:scrape:Ignoring malparsed backtrace: yield vars , File "/home/teuthworker/src/github.com_Matan-B_ceph_cb98f5a0f91af829569d87fce3d044ba583a0457/qa/tasks/ceph.py", line 1902, in task , ctx.managers[config['cluster']].stop_pg_num_changes() 2022-09-08T20:09:41.457 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T08:43:23.764 INFO:tasks.ceph.osd.3:Stopped , 2022-09-08T08:43:23.765 INFO:teuthology.misc:Shutting down mgr daemons... , 2022-09-08T08:43:23.766 DEBUG:tasks.ceph.mgr.x:waiting for process to exit 2022-09-08T20:09:41.458 WARNING:scrape:Ignoring malparsed backtrace: Traceback (most recent call last): , File "/home/teuthworker/src/git.ceph.com_git_teuthology_b58f2c18636eb10faa77ed3614abd00cb85dfc2c/teuthology/contextutil.py", line 33, in nested , yield vars 2022-09-08T20:09:41.459 WARNING:scrape:Ignoring malparsed backtrace: /usr/bin/which: unrecognized option '--objects' , /usr/bin/which: unrecognized option '--max-in-flight' , /usr/bin/which: unrecognized option '--size' 2022-09-08T20:09:41.564 WARNING:scrape:Ignoring malparsed backtrace: ceph version 17.0.0-14859-ge85e1217 (e85e121747baf9691b263effef4c284a96431457) quincy (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7fdd7aaffcb0] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x588) [0x561eef6fe3aa] 2022-09-08T20:09:41.567 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T08:23:47.788 INFO:tasks.ceph.ceph_manager.ceph:need seq 34359738397 got 34359738395 for osd.1 , 2022-09-08T08:23:47.830 INFO:teuthology.orchestra.run.smithi181.stdout:34359738395 , 2022-09-08T08:23:47.846 INFO:tasks.ceph.ceph_manager.ceph:need seq 34359738397 got 34359738395 for osd.2 2022-09-08T20:09:41.568 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T08:23:55.880 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2022-09-08T08:23:21.961791+0000 time.struct_time(tm_year=2022, tm_mon=9, tm_mday=8, tm_hour=8, tm_min=23, tm_sec=21, tm_wday=3, tm_yday=251, tm_isdst=-1) <= time.struct_time(tm_year=2022, tm_mon=9, tm_mday=8, tm_hour=8, tm_min=23, tm_sec=52, tm_wday=3, tm_yday=251, tm_isdst=0) , 2022-09-08T08:23:55.881 INFO:tasks.ceph:pgid 3.1 last_scrub_stamp 2022-09-08T08:23:28.327981+0000 time.struct_time(tm_year=2022, tm_mon=9, tm_mday=8, tm_hour=8, tm_min=23, tm_sec=28, tm_wday=3, tm_yday=251, tm_isdst=-1) <= time.struct_time(tm_year=2022, tm_mon=9, tm_mday=8, tm_hour=8, tm_min=23, tm_sec=52, tm_wday=3, tm_yday=251, tm_isdst=0) , 2022-09-08T08:23:55.881 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2022-09-08T08:23:21.961791+0000 time.struct_time(tm_year=2022, tm_mon=9, tm_mday=8, tm_hour=8, tm_min=23, tm_sec=21, tm_wday=3, tm_yday=251, tm_isdst=-1) <= time.struct_time(tm_year=2022, tm_mon=9, tm_mday=8, tm_hour=8, tm_min=23, tm_sec=52, tm_wday=3, tm_yday=251, tm_isdst=0) 2022-09-08T20:09:41.569 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T08:24:24.890 DEBUG:teuthology.misc:Transferring archived files from smithi186:/var/log/ceph to /home/teuthworker/archive/matan-2022-09-08_07:55:04-crimson-rados-main-distro-crimson-smithi/7020164/remote/smithi186/log , 2022-09-08T08:24:24.895 DEBUG:teuthology.orchestra.run.smithi186:> sudo tar cz -f - -C /var/log/ceph -- . , 2022-09-08T08:24:25.048 DEBUG:teuthology.run_tasks:Unwinding manager install 2022-09-08T20:09:41.570 WARNING:scrape:Ignoring malparsed backtrace: , 2022-09-08T08:24:42.318 INFO:teuthology.task.internal: dump_program: ceph_test_rados --max-ops 400000 --objects 1024 --max-in-flight 64 --size 40000 , 2022-09-08T08:24:42.319 DEBUG:teuthology.orchestra.run.smithi181:> which 'ceph_test_rados --max-ops 400000 --objects 1024 --max-in-flight 64 --size 40000' 2022-09-08T20:09:41.578 INFO:scrape:Found 2 distinct failure reasons 2022-09-08T20:09:41.579 INFO:scrape:Dead: b"2022-09-08T20:07:44.989 DEBUG:teuthology.orchestra.run.smithi116:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true" 2022-09-08T20:09:41.580 INFO:scrape:1 jobs: ['7020162'] 2022-09-08T20:09:41.580 INFO:scrape:suites: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{default}', 'centos_latest', 'clusters/{fixed-2}', 'crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'deploy/ceph', 'objectstore/bluestore', 'thrashers/default', 'thrashosds-health', 'workloads/small-objects}'] 2022-09-08T20:09:41.581 INFO:scrape: 2022-09-08T20:09:41.582 INFO:scrape:Failure: Command crashed: 'CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --max-ops 400000 --objects 1024 --max-in-flight 64 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 600 --op read 100 --op write 50 --op delete 10 --op write_excl 50 --pool unique_pool_0' 2022-09-08T20:09:41.582 INFO:scrape:2 jobs: ['7020163', '7020164'] 2022-09-08T20:09:41.583 INFO:scrape:suites intersection: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{default}', 'centos_latest', 'clusters/{fixed-2}', 'crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'deploy/ceph', 'objectstore/bluestore', 'thrashers/default', 'thrashosds-health', 'workloads/small-objects}'] 2022-09-08T20:09:41.583 INFO:scrape:suites union: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{default}', 'centos_latest', 'clusters/{fixed-2}', 'crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'deploy/ceph', 'objectstore/bluestore', 'thrashers/default', 'thrashosds-health', 'workloads/small-objects}'] 2022-09-08T20:09:41.584 INFO:scrape: