2022-08-25T08:33:08.613 INFO:root:teuthology version: 1.1.1.dev412+gb1d387f1 2022-08-25T08:33:08.631 INFO:teuthology.results:seed: '7286' 2022-08-25T08:36:16.439 INFO:root:teuthology version: 1.1.1.dev412+gb1d387f1 2022-08-25T08:36:16.457 INFO:teuthology.results:Waiting up to 43200 seconds for tests to finish... 2022-08-25T11:56:33.578 INFO:teuthology.results:Tests finished! gathering results... 2022-08-25T11:56:34.342 INFO:scrape:Found 9 jobs 2022-08-25T11:56:34.884 WARNING:scrape:Ignoring malparsed backtrace: ceph version 17.0.0-14583-g281beabd (281beabdfd6958897b81311e92181abac8827ef6) quincy (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7fcff9e30cb0] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x229) [0x55555707804b] 2022-08-25T11:56:34.886 WARNING:scrape:Ignoring malparsed backtrace: 2022-08-25T08:57:19.649 DEBUG:teuthology.orchestra.run.smithi120:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2022-08-25T08:57:19.970 INFO:teuthology.orchestra.run.smithi120.stdout: , dumped all 2022-08-25T11:56:34.931 WARNING:scrape:Ignoring malparsed backtrace: 2022-08-25T08:58:52.087 INFO:teuthology.orchestra.run.smithi120.stdout: , 2022-08-25T08:58:52.087 INFO:teuthology.orchestra.run.smithi120.stdout:{"epoch":65,"fsid":"d3b8cc72-a329-49ca-9891-9e338d193b22","created":"2022-08-25T08:55:36.285594+0000","modified":"2022-08-25T08:57:34.327920+0000","last_up_change":"2022-08-25T08:57:00.318215+0000","last_in_change":"2022-08-25T08:55:37.701279+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-08-25T08:55:43.945711+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-08-25T08:55:45.741738+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-08-25T08:56:00.053672+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"b9a77d8d-1953-4121-a610-6cd7c22cf19a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6800","nonce":99249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6801","nonce":99249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6805","nonce":99249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6804","nonce":99249}]},"public_addr":"172.21.15.120:6800/99249","cluster_addr":"172.21.15.120:6801/99249","heartbeat_back_addr":"172.21.15.120:6805/99249","heartbeat_front_addr":"172.21.15.120:6804/99249","state":["exists","up"]},{"osd":1,"uuid":"653ec5ae-eec8-4f72-8284-f4f38fd5b710","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":29,"down_at":47,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6803","nonce":99250}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6802","nonce":99250}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6807","nonce":99250}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6806","nonce":99250}]},"public_addr":"172.21.15.120:6803/99250","cluster_addr":"172.21.15.120:6802/99250","heartbeat_back_addr":"172.21.15.120:6807/99250","heartbeat_front_addr":"172.21.15.120:6806/99250","state":["exists"]},{"osd":2,"uuid":"43ffe4d8-e4a4-4969-b819-583c0034b9d9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6803","nonce":98654}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6802","nonce":98654}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6805","nonce":98654}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6806","nonce":98654}]},"public_addr":"172.21.15.190:6803/98654","cluster_addr":"172.21.15.190:6802/98654","heartbeat_back_addr":"172.21.15.190:6805/98654","heartbeat_front_addr":"172.21.15.190:6806/98654","state":["exists","up"]},{"osd":3,"uuid":"b3b5ae35-8ec6-4e53-867a-281ce5a2bc40","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6800","nonce":98655}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6801","nonce":98655}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6804","nonce":98655}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6807","nonce":98655}]},"public_addr":"172.21.15.190:6800/98655","cluster_addr":"172.21.15.190:6801/98655","heartbeat_back_addr":"172.21.15.190:6804/98655","heartbeat_front_addr":"172.21.15.190:6807/98655","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2022-08-25T08:57:00.318215+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[{"pgid":"3.5","osds":[2,3,1]},{"pgid":"3.e","osds":[0,3,1]}],"pg_upmap_items":[],"pg_temp":[{"pgid":"2.1","osds":[3,2,0]},{"pgid":"2.7","osds":[0,2,3]},{"pgid":"3.0","osds":[3,2,0]},{"pgid":"3.2","osds":[0,3,2]},{"pgid":"3.6","osds":[2,3,0]},{"pgid":"3.8","osds":[3,2,0]},{"pgid":"3.9","osds":[0,2,3]},{"pgid":"3.d","osds":[3,0,2]},{"pgid":"3.e","osds":[0,3,2]},{"pgid":"3.f","osds":[0,3,2]}],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2022-08-25T08:58:55.098 DEBUG:teuthology.orchestra.run.smithi120:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2022-08-25T11:56:34.936 WARNING:scrape:Ignoring malparsed backtrace: 2022-08-25T09:00:45.558 DEBUG:teuthology.orchestra.run.smithi120:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2022-08-25T09:00:45.880 INFO:teuthology.orchestra.run.smithi120.stdout: , 2022-08-25T09:00:45.881 INFO:teuthology.orchestra.run.smithi120.stdout:{"epoch":65,"fsid":"d3b8cc72-a329-49ca-9891-9e338d193b22","created":"2022-08-25T08:55:36.285594+0000","modified":"2022-08-25T08:57:34.327920+0000","last_up_change":"2022-08-25T08:57:00.318215+0000","last_in_change":"2022-08-25T08:55:37.701279+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-08-25T08:55:43.945711+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-08-25T08:55:45.741738+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-08-25T08:56:00.053672+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"b9a77d8d-1953-4121-a610-6cd7c22cf19a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6800","nonce":99249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6801","nonce":99249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6805","nonce":99249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6804","nonce":99249}]},"public_addr":"172.21.15.120:6800/99249","cluster_addr":"172.21.15.120:6801/99249","heartbeat_back_addr":"172.21.15.120:6805/99249","heartbeat_front_addr":"172.21.15.120:6804/99249","state":["exists","up"]},{"osd":1,"uuid":"653ec5ae-eec8-4f72-8284-f4f38fd5b710","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":29,"down_at":47,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6803","nonce":99250}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6802","nonce":99250}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6807","nonce":99250}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6806","nonce":99250}]},"public_addr":"172.21.15.120:6803/99250","cluster_addr":"172.21.15.120:6802/99250","heartbeat_back_addr":"172.21.15.120:6807/99250","heartbeat_front_addr":"172.21.15.120:6806/99250","state":["exists"]},{"osd":2,"uuid":"43ffe4d8-e4a4-4969-b819-583c0034b9d9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6803","nonce":98654}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6802","nonce":98654}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6805","nonce":98654}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6806","nonce":98654}]},"public_addr":"172.21.15.190:6803/98654","cluster_addr":"172.21.15.190:6802/98654","heartbeat_back_addr":"172.21.15.190:6805/98654","heartbeat_front_addr":"172.21.15.190:6806/98654","state":["exists","up"]},{"osd":3,"uuid":"b3b5ae35-8ec6-4e53-867a-281ce5a2bc40","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6800","nonce":98655}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6801","nonce":98655}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6804","nonce":98655}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6807","nonce":98655}]},"public_addr":"172.21.15.190:6800/98655","cluster_addr":"172.21.15.190:6801/98655","heartbeat_back_addr":"172.21.15.190:6804/98655","heartbeat_front_addr":"172.21.15.190:6807/98655","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2022-08-25T08:57:00.318215+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[{"pgid":"3.5","osds":[2,3,1]},{"pgid":"3.e","osds":[0,3,1]}],"pg_upmap_items":[],"pg_temp":[{"pgid":"2.1","osds":[3,2,0]},{"pgid":"2.7","osds":[0,2,3]},{"pgid":"3.0","osds":[3,2,0]},{"pgid":"3.2","osds":[0,3,2]},{"pgid":"3.6","osds":[2,3,0]},{"pgid":"3.8","osds":[3,2,0]},{"pgid":"3.9","osds":[0,2,3]},{"pgid":"3.d","osds":[3,0,2]},{"pgid":"3.e","osds":[0,3,2]},{"pgid":"3.f","osds":[0,3,2]}],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2022-08-25T11:56:34.939 WARNING:scrape:Ignoring malparsed backtrace: 2022-08-25T09:02:36.917 INFO:teuthology.orchestra.run.smithi120.stdout:{"epoch":65,"fsid":"d3b8cc72-a329-49ca-9891-9e338d193b22","created":"2022-08-25T08:55:36.285594+0000","modified":"2022-08-25T08:57:34.327920+0000","last_up_change":"2022-08-25T08:57:00.318215+0000","last_in_change":"2022-08-25T08:55:37.701279+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-08-25T08:55:43.945711+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-08-25T08:55:45.741738+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-08-25T08:56:00.053672+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"b9a77d8d-1953-4121-a610-6cd7c22cf19a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6800","nonce":99249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6801","nonce":99249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6805","nonce":99249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6804","nonce":99249}]},"public_addr":"172.21.15.120:6800/99249","cluster_addr":"172.21.15.120:6801/99249","heartbeat_back_addr":"172.21.15.120:6805/99249","heartbeat_front_addr":"172.21.15.120:6804/99249","state":["exists","up"]},{"osd":1,"uuid":"653ec5ae-eec8-4f72-8284-f4f38fd5b710","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":29,"down_at":47,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6803","nonce":99250}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6802","nonce":99250}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6807","nonce":99250}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6806","nonce":99250}]},"public_addr":"172.21.15.120:6803/99250","cluster_addr":"172.21.15.120:6802/99250","heartbeat_back_addr":"172.21.15.120:6807/99250","heartbeat_front_addr":"172.21.15.120:6806/99250","state":["exists"]},{"osd":2,"uuid":"43ffe4d8-e4a4-4969-b819-583c0034b9d9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6803","nonce":98654}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6802","nonce":98654}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6805","nonce":98654}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6806","nonce":98654}]},"public_addr":"172.21.15.190:6803/98654","cluster_addr":"172.21.15.190:6802/98654","heartbeat_back_addr":"172.21.15.190:6805/98654","heartbeat_front_addr":"172.21.15.190:6806/98654","state":["exists","up"]},{"osd":3,"uuid":"b3b5ae35-8ec6-4e53-867a-281ce5a2bc40","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6800","nonce":98655}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6801","nonce":98655}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6804","nonce":98655}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6807","nonce":98655}]},"public_addr":"172.21.15.190:6800/98655","cluster_addr":"172.21.15.190:6801/98655","heartbeat_back_addr":"172.21.15.190:6804/98655","heartbeat_front_addr":"172.21.15.190:6807/98655","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2022-08-25T08:57:00.318215+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[{"pgid":"3.5","osds":[2,3,1]},{"pgid":"3.e","osds":[0,3,1]}],"pg_upmap_items":[],"pg_temp":[{"pgid":"2.1","osds":[3,2,0]},{"pgid":"2.7","osds":[0,2,3]},{"pgid":"3.0","osds":[3,2,0]},{"pgid":"3.2","osds":[0,3,2]},{"pgid":"3.6","osds":[2,3,0]},{"pgid":"3.8","osds":[3,2,0]},{"pgid":"3.9","osds":[0,2,3]},{"pgid":"3.d","osds":[3,0,2]},{"pgid":"3.e","osds":[0,3,2]},{"pgid":"3.f","osds":[0,3,2]}],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2022-08-25T09:02:39.934 DEBUG:teuthology.orchestra.run.smithi120:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2022-08-25T09:02:40.258 INFO:teuthology.orchestra.run.smithi120.stdout: 2022-08-25T11:56:34.941 WARNING:scrape:Ignoring malparsed backtrace: 2022-08-25T09:04:30.755 INFO:teuthology.orchestra.run.smithi120.stdout: , 2022-08-25T09:04:30.756 INFO:teuthology.orchestra.run.smithi120.stdout:{"epoch":65,"fsid":"d3b8cc72-a329-49ca-9891-9e338d193b22","created":"2022-08-25T08:55:36.285594+0000","modified":"2022-08-25T08:57:34.327920+0000","last_up_change":"2022-08-25T08:57:00.318215+0000","last_in_change":"2022-08-25T08:55:37.701279+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-08-25T08:55:43.945711+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-08-25T08:55:45.741738+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-08-25T08:56:00.053672+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"b9a77d8d-1953-4121-a610-6cd7c22cf19a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6800","nonce":99249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6801","nonce":99249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6805","nonce":99249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6804","nonce":99249}]},"public_addr":"172.21.15.120:6800/99249","cluster_addr":"172.21.15.120:6801/99249","heartbeat_back_addr":"172.21.15.120:6805/99249","heartbeat_front_addr":"172.21.15.120:6804/99249","state":["exists","up"]},{"osd":1,"uuid":"653ec5ae-eec8-4f72-8284-f4f38fd5b710","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":29,"down_at":47,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6803","nonce":99250}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6802","nonce":99250}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6807","nonce":99250}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6806","nonce":99250}]},"public_addr":"172.21.15.120:6803/99250","cluster_addr":"172.21.15.120:6802/99250","heartbeat_back_addr":"172.21.15.120:6807/99250","heartbeat_front_addr":"172.21.15.120:6806/99250","state":["exists"]},{"osd":2,"uuid":"43ffe4d8-e4a4-4969-b819-583c0034b9d9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6803","nonce":98654}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6802","nonce":98654}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6805","nonce":98654}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6806","nonce":98654}]},"public_addr":"172.21.15.190:6803/98654","cluster_addr":"172.21.15.190:6802/98654","heartbeat_back_addr":"172.21.15.190:6805/98654","heartbeat_front_addr":"172.21.15.190:6806/98654","state":["exists","up"]},{"osd":3,"uuid":"b3b5ae35-8ec6-4e53-867a-281ce5a2bc40","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6800","nonce":98655}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6801","nonce":98655}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6804","nonce":98655}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6807","nonce":98655}]},"public_addr":"172.21.15.190:6800/98655","cluster_addr":"172.21.15.190:6801/98655","heartbeat_back_addr":"172.21.15.190:6804/98655","heartbeat_front_addr":"172.21.15.190:6807/98655","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2022-08-25T08:57:00.318215+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[{"pgid":"3.5","osds":[2,3,1]},{"pgid":"3.e","osds":[0,3,1]}],"pg_upmap_items":[],"pg_temp":[{"pgid":"2.1","osds":[3,2,0]},{"pgid":"2.7","osds":[0,2,3]},{"pgid":"3.0","osds":[3,2,0]},{"pgid":"3.2","osds":[0,3,2]},{"pgid":"3.6","osds":[2,3,0]},{"pgid":"3.8","osds":[3,2,0]},{"pgid":"3.9","osds":[0,2,3]},{"pgid":"3.d","osds":[3,0,2]},{"pgid":"3.e","osds":[0,3,2]},{"pgid":"3.f","osds":[0,3,2]}],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2022-08-25T09:04:33.770 DEBUG:teuthology.orchestra.run.smithi120:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2022-08-25T11:56:34.943 WARNING:scrape:Ignoring malparsed backtrace: 2022-08-25T09:06:24.932 DEBUG:teuthology.orchestra.run.smithi120:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2022-08-25T09:06:25.255 INFO:teuthology.orchestra.run.smithi120.stdout: , 2022-08-25T09:06:25.255 INFO:teuthology.orchestra.run.smithi120.stdout:{"epoch":65,"fsid":"d3b8cc72-a329-49ca-9891-9e338d193b22","created":"2022-08-25T08:55:36.285594+0000","modified":"2022-08-25T08:57:34.327920+0000","last_up_change":"2022-08-25T08:57:00.318215+0000","last_in_change":"2022-08-25T08:55:37.701279+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-08-25T08:55:43.945711+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-08-25T08:55:45.741738+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-08-25T08:56:00.053672+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"b9a77d8d-1953-4121-a610-6cd7c22cf19a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6800","nonce":99249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6801","nonce":99249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6805","nonce":99249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6804","nonce":99249}]},"public_addr":"172.21.15.120:6800/99249","cluster_addr":"172.21.15.120:6801/99249","heartbeat_back_addr":"172.21.15.120:6805/99249","heartbeat_front_addr":"172.21.15.120:6804/99249","state":["exists","up"]},{"osd":1,"uuid":"653ec5ae-eec8-4f72-8284-f4f38fd5b710","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":29,"down_at":47,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6803","nonce":99250}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6802","nonce":99250}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6807","nonce":99250}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6806","nonce":99250}]},"public_addr":"172.21.15.120:6803/99250","cluster_addr":"172.21.15.120:6802/99250","heartbeat_back_addr":"172.21.15.120:6807/99250","heartbeat_front_addr":"172.21.15.120:6806/99250","state":["exists"]},{"osd":2,"uuid":"43ffe4d8-e4a4-4969-b819-583c0034b9d9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6803","nonce":98654}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6802","nonce":98654}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6805","nonce":98654}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6806","nonce":98654}]},"public_addr":"172.21.15.190:6803/98654","cluster_addr":"172.21.15.190:6802/98654","heartbeat_back_addr":"172.21.15.190:6805/98654","heartbeat_front_addr":"172.21.15.190:6806/98654","state":["exists","up"]},{"osd":3,"uuid":"b3b5ae35-8ec6-4e53-867a-281ce5a2bc40","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6800","nonce":98655}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6801","nonce":98655}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6804","nonce":98655}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6807","nonce":98655}]},"public_addr":"172.21.15.190:6800/98655","cluster_addr":"172.21.15.190:6801/98655","heartbeat_back_addr":"172.21.15.190:6804/98655","heartbeat_front_addr":"172.21.15.190:6807/98655","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2022-08-25T08:57:00.318215+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[{"pgid":"3.5","osds":[2,3,1]},{"pgid":"3.e","osds":[0,3,1]}],"pg_upmap_items":[],"pg_temp":[{"pgid":"2.1","osds":[3,2,0]},{"pgid":"2.7","osds":[0,2,3]},{"pgid":"3.0","osds":[3,2,0]},{"pgid":"3.2","osds":[0,3,2]},{"pgid":"3.6","osds":[2,3,0]},{"pgid":"3.8","osds":[3,2,0]},{"pgid":"3.9","osds":[0,2,3]},{"pgid":"3.d","osds":[3,0,2]},{"pgid":"3.e","osds":[0,3,2]},{"pgid":"3.f","osds":[0,3,2]}],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2022-08-25T11:56:34.944 WARNING:scrape:Ignoring malparsed backtrace: kernel callstack: , kernel callstack: , kernel callstack: 2022-08-25T11:56:34.946 WARNING:scrape:Ignoring malparsed backtrace: 600000000000-602000000000 ---p 00000000 00:00 0 , 602000000000-6020003d0000 rw-p 00000000 00:00 0 , 6020003d0000-602e00000000 ---p 00000000 00:00 0 2022-08-25T11:56:34.947 WARNING:scrape:Ignoring malparsed backtrace: 61d000000000-61d000660000 rw-p 00000000 00:00 0 , 61d000660000-61de00000000 ---p 00000000 00:00 0 , 61de00000000-61de00010000 rw-p 00000000 00:00 0 2022-08-25T11:56:34.948 WARNING:scrape:Ignoring malparsed backtrace: 7f6505a50000-7f6505af5000 rw-p 00000000 00:00 0 , 7f650610b000-7f65062d2000 rw-p 00000000 00:00 0 , 7f65062d7000-7f65064f3000 rw-p 00000000 00:00 0 2022-08-25T11:56:34.949 WARNING:scrape:Ignoring malparsed backtrace: 7f6514601000-7f6514645000 rw-p 00000000 00:00 0 , 7f6514672000-7f651469f000 rw-p 00000000 00:00 0 , 7f6514781000-7f65147ae000 rw-p 00000000 00:00 0 2022-08-25T11:56:34.950 WARNING:scrape:Ignoring malparsed backtrace: 7f651705b000-7f651707b000 rw-p 00000000 00:00 0 , 7f6517082000-7f6517092000 rw-p 00000000 00:00 0 , 7f6517098000-7f65170fc000 rw-p 00000000 00:00 0 2022-08-25T11:56:34.951 WARNING:scrape:Ignoring malparsed backtrace: 7f651f70f000-7f651f710000 rw-p 0007d000 08:01 262553 /usr/lib64/rados-classes/libcls_rbd.so.1.0.0 , 7f651f710000-7f651f716000 r--p 00000000 08:01 262544 /usr/lib64/rados-classes/libcls_numops.so.1.0.0 , 7f651f716000-7f651f71b000 r-xp 00006000 08:01 262544 /usr/lib64/rados-classes/libcls_numops.so.1.0.0 2022-08-25T11:56:34.951 WARNING:scrape:Ignoring malparsed backtrace: 7f651feb9000-7f651fec0000 r--p 00000000 08:01 262529 /usr/lib64/rados-classes/libcls_hello.so.1.0.0 , 7f651fec0000-7f651fec6000 r-xp 00007000 08:01 262529 /usr/lib64/rados-classes/libcls_hello.so.1.0.0 , 7f651fec6000-7f651fec8000 r--p 0000d000 08:01 262529 /usr/lib64/rados-classes/libcls_hello.so.1.0.0 2022-08-25T11:56:34.952 WARNING:scrape:Ignoring malparsed backtrace: 7f6531ed6000-7f65320d5000 ---p 00017000 08:01 15701 /usr/lib64/libgcc_s-8-20210514.so.1 , 7f65320d5000-7f65320d6000 r--p 00016000 08:01 15701 /usr/lib64/libgcc_s-8-20210514.so.1 , 7f65320d6000-7f65320d7000 rw-p 00017000 08:01 15701 /usr/lib64/libgcc_s-8-20210514.so.1 2022-08-25T11:56:34.953 WARNING:scrape:Ignoring malparsed backtrace: 7f6536f09000-7f6536f0a000 r--p 0002d000 08:01 8926 /usr/lib64/ld-2.28.so , 7f6536f0a000-7f6536f0c000 rw-p 0002e000 08:01 8926 /usr/lib64/ld-2.28.so , 7ffd3329e000-7ffd3334f000 rw-p 00000000 00:00 0 [stack] 2022-08-25T11:56:34.954 WARNING:scrape:Ignoring malparsed backtrace: 2022-08-25T09:08:31.905 DEBUG:teuthology.orchestra.run.smithi120:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2022-08-25T09:08:32.226 INFO:teuthology.orchestra.run.smithi120.stdout: , 2022-08-25T09:08:32.227 INFO:teuthology.orchestra.run.smithi120.stdout:{"epoch":76,"fsid":"d3b8cc72-a329-49ca-9891-9e338d193b22","created":"2022-08-25T08:55:36.285594+0000","modified":"2022-08-25T09:07:36.357164+0000","last_up_change":"2022-08-25T09:07:29.385566+0000","last_in_change":"2022-08-25T09:07:01.350997+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-08-25T08:55:43.945711+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-08-25T08:55:45.741738+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-08-25T08:56:00.053672+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"b9a77d8d-1953-4121-a610-6cd7c22cf19a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":74,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6800","nonce":99249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6801","nonce":99249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6805","nonce":99249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6804","nonce":99249}]},"public_addr":"172.21.15.120:6800/99249","cluster_addr":"172.21.15.120:6801/99249","heartbeat_back_addr":"172.21.15.120:6805/99249","heartbeat_front_addr":"172.21.15.120:6804/99249","state":["exists","up"]},{"osd":1,"uuid":"653ec5ae-eec8-4f72-8284-f4f38fd5b710","up":0,"in":0,"weight":0,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":29,"down_at":47,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6803","nonce":99250}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6802","nonce":99250}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6807","nonce":99250}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6806","nonce":99250}]},"public_addr":"172.21.15.120:6803/99250","cluster_addr":"172.21.15.120:6802/99250","heartbeat_back_addr":"172.21.15.120:6807/99250","heartbeat_front_addr":"172.21.15.120:6806/99250","state":["autoout","exists"]},{"osd":2,"uuid":"43ffe4d8-e4a4-4969-b819-583c0034b9d9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":74,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6803","nonce":98654}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6802","nonce":98654}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6805","nonce":98654}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6806","nonce":98654}]},"public_addr":"172.21.15.190:6803/98654","cluster_addr":"172.21.15.190:6802/98654","heartbeat_back_addr":"172.21.15.190:6805/98654","heartbeat_front_addr":"172.21.15.190:6806/98654","state":["exists","up"]},{"osd":3,"uuid":"b3b5ae35-8ec6-4e53-867a-281ce5a2bc40","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":67,"down_at":73,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6800","nonce":98655}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6801","nonce":98655}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6804","nonce":98655}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6807","nonce":98655}]},"public_addr":"172.21.15.190:6800/98655","cluster_addr":"172.21.15.190:6801/98655","heartbeat_back_addr":"172.21.15.190:6804/98655","heartbeat_front_addr":"172.21.15.190:6807/98655","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2022-08-25T08:57:00.318215+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":65536,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2022-08-25T09:07:29.385566+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[{"pgid":"3.5","osds":[2,3,1]},{"pgid":"3.e","osds":[0,3,1]}],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2022-08-25T11:56:34.955 WARNING:scrape:Ignoring malparsed backtrace: 2022-08-25T09:10:05.960 DEBUG:teuthology.orchestra.run.smithi120:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2022-08-25T09:10:06.285 INFO:teuthology.orchestra.run.smithi120.stdout: , 2022-08-25T09:10:06.285 INFO:teuthology.orchestra.run.smithi120.stdout:{"epoch":76,"fsid":"d3b8cc72-a329-49ca-9891-9e338d193b22","created":"2022-08-25T08:55:36.285594+0000","modified":"2022-08-25T09:07:36.357164+0000","last_up_change":"2022-08-25T09:07:29.385566+0000","last_in_change":"2022-08-25T09:07:01.350997+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-08-25T08:55:43.945711+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-08-25T08:55:45.741738+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-08-25T08:56:00.053672+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"b9a77d8d-1953-4121-a610-6cd7c22cf19a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":74,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6800","nonce":99249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6801","nonce":99249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6805","nonce":99249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6804","nonce":99249}]},"public_addr":"172.21.15.120:6800/99249","cluster_addr":"172.21.15.120:6801/99249","heartbeat_back_addr":"172.21.15.120:6805/99249","heartbeat_front_addr":"172.21.15.120:6804/99249","state":["exists","up"]},{"osd":1,"uuid":"653ec5ae-eec8-4f72-8284-f4f38fd5b710","up":0,"in":0,"weight":0,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":29,"down_at":47,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6803","nonce":99250}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6802","nonce":99250}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6807","nonce":99250}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6806","nonce":99250}]},"public_addr":"172.21.15.120:6803/99250","cluster_addr":"172.21.15.120:6802/99250","heartbeat_back_addr":"172.21.15.120:6807/99250","heartbeat_front_addr":"172.21.15.120:6806/99250","state":["autoout","exists"]},{"osd":2,"uuid":"43ffe4d8-e4a4-4969-b819-583c0034b9d9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":74,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6803","nonce":98654}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6802","nonce":98654}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6805","nonce":98654}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6806","nonce":98654}]},"public_addr":"172.21.15.190:6803/98654","cluster_addr":"172.21.15.190:6802/98654","heartbeat_back_addr":"172.21.15.190:6805/98654","heartbeat_front_addr":"172.21.15.190:6806/98654","state":["exists","up"]},{"osd":3,"uuid":"b3b5ae35-8ec6-4e53-867a-281ce5a2bc40","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":67,"down_at":73,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6800","nonce":98655}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6801","nonce":98655}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6804","nonce":98655}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6807","nonce":98655}]},"public_addr":"172.21.15.190:6800/98655","cluster_addr":"172.21.15.190:6801/98655","heartbeat_back_addr":"172.21.15.190:6804/98655","heartbeat_front_addr":"172.21.15.190:6807/98655","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2022-08-25T08:57:00.318215+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":65536,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2022-08-25T09:07:29.385566+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[{"pgid":"3.5","osds":[2,3,1]},{"pgid":"3.e","osds":[0,3,1]}],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2022-08-25T11:56:34.956 WARNING:scrape:Ignoring malparsed backtrace: 2022-08-25T09:11:39.802 DEBUG:teuthology.orchestra.run.smithi120:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2022-08-25T09:11:40.121 INFO:teuthology.orchestra.run.smithi120.stdout: , 2022-08-25T09:11:40.122 INFO:teuthology.orchestra.run.smithi120.stdout:{"epoch":76,"fsid":"d3b8cc72-a329-49ca-9891-9e338d193b22","created":"2022-08-25T08:55:36.285594+0000","modified":"2022-08-25T09:07:36.357164+0000","last_up_change":"2022-08-25T09:07:29.385566+0000","last_in_change":"2022-08-25T09:07:01.350997+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-08-25T08:55:43.945711+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-08-25T08:55:45.741738+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-08-25T08:56:00.053672+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"b9a77d8d-1953-4121-a610-6cd7c22cf19a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":74,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6800","nonce":99249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6801","nonce":99249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6805","nonce":99249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6804","nonce":99249}]},"public_addr":"172.21.15.120:6800/99249","cluster_addr":"172.21.15.120:6801/99249","heartbeat_back_addr":"172.21.15.120:6805/99249","heartbeat_front_addr":"172.21.15.120:6804/99249","state":["exists","up"]},{"osd":1,"uuid":"653ec5ae-eec8-4f72-8284-f4f38fd5b710","up":0,"in":0,"weight":0,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":29,"down_at":47,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6803","nonce":99250}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6802","nonce":99250}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6807","nonce":99250}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.120:6806","nonce":99250}]},"public_addr":"172.21.15.120:6803/99250","cluster_addr":"172.21.15.120:6802/99250","heartbeat_back_addr":"172.21.15.120:6807/99250","heartbeat_front_addr":"172.21.15.120:6806/99250","state":["autoout","exists"]},{"osd":2,"uuid":"43ffe4d8-e4a4-4969-b819-583c0034b9d9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":74,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6803","nonce":98654}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6802","nonce":98654}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6805","nonce":98654}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6806","nonce":98654}]},"public_addr":"172.21.15.190:6803/98654","cluster_addr":"172.21.15.190:6802/98654","heartbeat_back_addr":"172.21.15.190:6805/98654","heartbeat_front_addr":"172.21.15.190:6806/98654","state":["exists","up"]},{"osd":3,"uuid":"b3b5ae35-8ec6-4e53-867a-281ce5a2bc40","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":67,"down_at":73,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6800","nonce":98655}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6801","nonce":98655}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6804","nonce":98655}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.190:6807","nonce":98655}]},"public_addr":"172.21.15.190:6800/98655","cluster_addr":"172.21.15.190:6801/98655","heartbeat_back_addr":"172.21.15.190:6804/98655","heartbeat_front_addr":"172.21.15.190:6807/98655","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2022-08-25T08:57:00.318215+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":65536,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2022-08-25T09:07:29.385566+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[{"pgid":"3.5","osds":[2,3,1]},{"pgid":"3.e","osds":[0,3,1]}],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2022-08-25T11:56:34.957 WARNING:scrape:Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_git_teuthology_b1d387f12b117399cb87c86aaa341398fa0c0919/teuthology/orchestra/run.py", line 183, in _raise_for_status , node=self.hostname, label=self.label , teuthology.exceptions.CommandFailedError: Command failed on smithi120 with status 124: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json' 2022-08-25T11:56:34.958 WARNING:scrape:Ignoring malparsed backtrace: yield vars , File "/home/teuthworker/src/github.com_ceph_ceph-c_281beabdfd6958897b81311e92181abac8827ef6/qa/tasks/ceph.py", line 1902, in task , ctx.managers[config['cluster']].stop_pg_num_changes() 2022-08-25T11:56:34.959 WARNING:scrape:Ignoring malparsed backtrace: self._raise_for_status() , File "/home/teuthworker/src/git.ceph.com_git_teuthology_b1d387f12b117399cb87c86aaa341398fa0c0919/teuthology/orchestra/run.py", line 179, in _raise_for_status , raise CommandCrashedError(command=self.command) 2022-08-25T11:56:34.959 WARNING:scrape:Ignoring malparsed backtrace: 2022-08-25T09:16:30.034 DEBUG:teuthology.orchestra.run.smithi190:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 , 2022-08-25T09:16:30.063 INFO:teuthology.task.internal.syslog:Compressing syslogs... , 2022-08-25T09:16:30.064 DEBUG:teuthology.orchestra.run.smithi120:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2022-08-25T11:56:35.255 ERROR:teuthology.results:error generating memo/results Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_git_teuthology_b1d387f12b117399cb87c86aaa341398fa0c0919/teuthology/results.py", line 34, in main int(args['--timeout']), args['--dry-run']) File "/home/teuthworker/src/git.ceph.com_git_teuthology_b1d387f12b117399cb87c86aaa341398fa0c0919/teuthology/results.py", line 71, in results Scraper(archive_dir).analyze() File "/home/teuthworker/src/git.ceph.com_git_teuthology_b1d387f12b117399cb87c86aaa341398fa0c0919/teuthology/scrape.py", line 490, in analyze if reason.match(job): File "/home/teuthworker/src/git.ceph.com_git_teuthology_b1d387f12b117399cb87c86aaa341398fa0c0919/teuthology/scrape.py", line 75, in match if (self.backtrace is None) != (job.get_backtrace() is None): File "/home/teuthworker/src/git.ceph.com_git_teuthology_b1d387f12b117399cb87c86aaa341398fa0c0919/teuthology/scrape.py", line 338, in get_backtrace self._populate_backtrace() File "/home/teuthworker/src/git.ceph.com_git_teuthology_b1d387f12b117399cb87c86aaa341398fa0c0919/teuthology/scrape.py", line 384, in _populate_backtrace bt, ass = self._search_backtrace(GzipFile(gzipped_log_path)) File "/home/teuthworker/src/git.ceph.com_git_teuthology_b1d387f12b117399cb87c86aaa341398fa0c0919/teuthology/scrape.py", line 304, in _search_backtrace if ".stderr:" in line: TypeError: a bytes-like object is required, not 'str'