2020-08-22T03:34:27.659 INFO:root:teuthology version: 1.0.0-c0ecc52 2020-08-22T03:34:27.662 INFO:teuthology.results:seed: '2477' 2020-08-22T03:35:57.749 INFO:root:teuthology version: 1.0.0-c0ecc52 2020-08-22T03:35:57.751 INFO:teuthology.results:Waiting up to 43200 seconds for tests to finish... 2020-08-22T03:56:58.345 INFO:teuthology.results:Tests finished! gathering results... 2020-08-22T03:56:58.418 INFO:scrape:Found 1 jobs 2020-08-22T03:56:58.447 WARNING:scrape:Ignoring malparsed backtrace: ceph version 16.0.0-4564-ge64d8132f64 (e64d8132f648167078f5841ad65b42669a68d2cb) pacific (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe0) [0x7ff89e69a39e] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x67d) [0x55dea91d2add] 2020-08-22T03:56:58.450 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:50:14.250 INFO:teuthology.orchestra.run.smithi183.stdout: , 2020-08-22T03:50:14.250 INFO:teuthology.orchestra.run.smithi183.stdout:{"epoch":32,"fsid":"9972e83e-5caf-455d-81f6-10e7c5bcb3e2","created":"2020-08-22T03:49:05.967889+0000","modified":"2020-08-22T03:50:11.943469+0000","last_up_change":"2020-08-22T03:49:42.013320+0000","last_in_change":"2020-08-22T03:49:11.484438+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-22T03:49:09.511038+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-22T03:49:16.052382+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-22T03:49:28.065828+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"e0d320b8-4993-4b1a-b741-ab771add082d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6800","nonce":25504}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6801","nonce":25504}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6806","nonce":25504}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6807","nonce":25504}]},"public_addr":"172.21.15.183:6800/25504","cluster_addr":"172.21.15.183:6801/25504","heartbeat_back_addr":"172.21.15.183:6806/25504","heartbeat_front_addr":"172.21.15.183:6807/25504","state":["exists","up"]},{"osd":1,"uuid":"51144378-a6ca-485c-8a15-20c001a1ed60","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":18,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6802","nonce":25547}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6803","nonce":25547}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6804","nonce":25547}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6805","nonce":25547}]},"public_addr":"172.21.15.183:6802/25547","cluster_addr":"172.21.15.183:6803/25547","heartbeat_back_addr":"172.21.15.183:6804/25547","heartbeat_front_addr":"172.21.15.183:6805/25547","state":["exists"]},{"osd":2,"uuid":"9f4f44e0-4c21-49c7-83c6-9fd9c902a564","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":23,"up_from":24,"up_thru":24,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6802","nonce":25353}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6803","nonce":25353}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6806","nonce":25353}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6807","nonce":25353}]},"public_addr":"172.21.15.92:6802/25353","cluster_addr":"172.21.15.92:6803/25353","heartbeat_back_addr":"172.21.15.92:6806/25353","heartbeat_front_addr":"172.21.15.92:6807/25353","state":["exists","up"]},{"osd":3,"uuid":"39fb66cd-8671-4a52-a6bd-6376d5ff4f5b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":23,"up_from":24,"up_thru":31,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6804","nonce":25397}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6805","nonce":25397}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6808","nonce":25397}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6809","nonce":25397}]},"public_addr":"172.21.15.92:6804/25397","cluster_addr":"172.21.15.92:6805/25397","heartbeat_back_addr":"172.21.15.92:6808/25397","heartbeat_front_addr":"172.21.15.92:6809/25397","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-22T03:49:41.010632+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"2020-08-22T03:49:41.010632+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-22T03:49:41.010632+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.2","mappings":[{"from":0,"to":3}]},{"pgid":"2.4","mappings":[{"from":2,"to":1}]},{"pgid":"2.7","mappings":[{"from":2,"to":3}]}],"pg_temp":[{"pgid":"2.2","osds":[3,2,0]},{"pgid":"2.4","osds":[3,0,2]},{"pgid":"2.7","osds":[0,3,2]}],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-22T03:50:15.942+0000 7f343b7fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.0 v2:172.21.15.183:6800/25504 for 24sec e20 v20) 2020-08-22T03:56:58.453 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:51:43.036 INFO:teuthology.orchestra.run.smithi183.stdout:{"epoch":32,"fsid":"9972e83e-5caf-455d-81f6-10e7c5bcb3e2","created":"2020-08-22T03:49:05.967889+0000","modified":"2020-08-22T03:50:11.943469+0000","last_up_change":"2020-08-22T03:49:42.013320+0000","last_in_change":"2020-08-22T03:49:11.484438+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-22T03:49:09.511038+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-22T03:49:16.052382+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-22T03:49:28.065828+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"e0d320b8-4993-4b1a-b741-ab771add082d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6800","nonce":25504}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6801","nonce":25504}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6806","nonce":25504}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6807","nonce":25504}]},"public_addr":"172.21.15.183:6800/25504","cluster_addr":"172.21.15.183:6801/25504","heartbeat_back_addr":"172.21.15.183:6806/25504","heartbeat_front_addr":"172.21.15.183:6807/25504","state":["exists","up"]},{"osd":1,"uuid":"51144378-a6ca-485c-8a15-20c001a1ed60","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":18,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6802","nonce":25547}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6803","nonce":25547}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6804","nonce":25547}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6805","nonce":25547}]},"public_addr":"172.21.15.183:6802/25547","cluster_addr":"172.21.15.183:6803/25547","heartbeat_back_addr":"172.21.15.183:6804/25547","heartbeat_front_addr":"172.21.15.183:6805/25547","state":["exists"]},{"osd":2,"uuid":"9f4f44e0-4c21-49c7-83c6-9fd9c902a564","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":23,"up_from":24,"up_thru":24,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6802","nonce":25353}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6803","nonce":25353}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6806","nonce":25353}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6807","nonce":25353}]},"public_addr":"172.21.15.92:6802/25353","cluster_addr":"172.21.15.92:6803/25353","heartbeat_back_addr":"172.21.15.92:6806/25353","heartbeat_front_addr":"172.21.15.92:6807/25353","state":["exists","up"]},{"osd":3,"uuid":"39fb66cd-8671-4a52-a6bd-6376d5ff4f5b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":23,"up_from":24,"up_thru":31,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6804","nonce":25397}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6805","nonce":25397}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6808","nonce":25397}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6809","nonce":25397}]},"public_addr":"172.21.15.92:6804/25397","cluster_addr":"172.21.15.92:6805/25397","heartbeat_back_addr":"172.21.15.92:6808/25397","heartbeat_front_addr":"172.21.15.92:6809/25397","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-22T03:49:41.010632+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"2020-08-22T03:49:41.010632+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-22T03:49:41.010632+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.2","mappings":[{"from":0,"to":3}]},{"pgid":"2.4","mappings":[{"from":2,"to":1}]},{"pgid":"2.7","mappings":[{"from":2,"to":3}]}],"pg_temp":[{"pgid":"2.2","osds":[3,2,0]},{"pgid":"2.4","osds":[3,0,2]},{"pgid":"2.7","osds":[0,3,2]}],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-22T03:51:45.951+0000 7f343b7fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.0 v2:172.21.15.183:6800/25504 for 24sec e20 v20) , 2020-08-22T03:51:46.046 INFO:teuthology.orchestra.run.smithi183:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json 2020-08-22T03:56:58.456 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:53:11.768 INFO:teuthology.orchestra.run.smithi183.stdout:{"epoch":32,"fsid":"9972e83e-5caf-455d-81f6-10e7c5bcb3e2","created":"2020-08-22T03:49:05.967889+0000","modified":"2020-08-22T03:50:11.943469+0000","last_up_change":"2020-08-22T03:49:42.013320+0000","last_in_change":"2020-08-22T03:49:11.484438+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-22T03:49:09.511038+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-22T03:49:16.052382+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-22T03:49:28.065828+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"e0d320b8-4993-4b1a-b741-ab771add082d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6800","nonce":25504}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6801","nonce":25504}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6806","nonce":25504}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6807","nonce":25504}]},"public_addr":"172.21.15.183:6800/25504","cluster_addr":"172.21.15.183:6801/25504","heartbeat_back_addr":"172.21.15.183:6806/25504","heartbeat_front_addr":"172.21.15.183:6807/25504","state":["exists","up"]},{"osd":1,"uuid":"51144378-a6ca-485c-8a15-20c001a1ed60","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":18,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6802","nonce":25547}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6803","nonce":25547}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6804","nonce":25547}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6805","nonce":25547}]},"public_addr":"172.21.15.183:6802/25547","cluster_addr":"172.21.15.183:6803/25547","heartbeat_back_addr":"172.21.15.183:6804/25547","heartbeat_front_addr":"172.21.15.183:6805/25547","state":["exists"]},{"osd":2,"uuid":"9f4f44e0-4c21-49c7-83c6-9fd9c902a564","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":23,"up_from":24,"up_thru":24,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6802","nonce":25353}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6803","nonce":25353}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6806","nonce":25353}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6807","nonce":25353}]},"public_addr":"172.21.15.92:6802/25353","cluster_addr":"172.21.15.92:6803/25353","heartbeat_back_addr":"172.21.15.92:6806/25353","heartbeat_front_addr":"172.21.15.92:6807/25353","state":["exists","up"]},{"osd":3,"uuid":"39fb66cd-8671-4a52-a6bd-6376d5ff4f5b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":23,"up_from":24,"up_thru":31,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6804","nonce":25397}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6805","nonce":25397}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6808","nonce":25397}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6809","nonce":25397}]},"public_addr":"172.21.15.92:6804/25397","cluster_addr":"172.21.15.92:6805/25397","heartbeat_back_addr":"172.21.15.92:6808/25397","heartbeat_front_addr":"172.21.15.92:6809/25397","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-22T03:49:41.010632+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"2020-08-22T03:49:41.010632+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-22T03:49:41.010632+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.2","mappings":[{"from":0,"to":3}]},{"pgid":"2.4","mappings":[{"from":2,"to":1}]},{"pgid":"2.7","mappings":[{"from":2,"to":3}]}],"pg_temp":[{"pgid":"2.2","osds":[3,2,0]},{"pgid":"2.4","osds":[3,0,2]},{"pgid":"2.7","osds":[0,3,2]}],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-22T03:53:14.780 INFO:teuthology.orchestra.run.smithi183:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-22T03:53:15.052 INFO:teuthology.orchestra.run.smithi183.stdout: 2020-08-22T03:56:58.461 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:54:17.718 INFO:teuthology.orchestra.run.smithi183.stdout: , dumped all , 2020-08-22T03:54:17.732 INFO:teuthology.orchestra.run.smithi183.stdout:{"pg_ready":true,"pg_map":{"version":183,"stamp":"2020-08-22T03:54:17.521444+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":827,"ondisk_log_size":827,"up":75,"acting":75,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":0,"num_osds":4,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"3.f","version":"19'50","reported_seq":"58","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:14.607086+0000","last_change":"2020-08-22T03:54:14.607086+0000","last_active":"2020-08-22T03:54:14.607086+0000","last_peered":"2020-08-22T03:54:14.607086+0000","last_clean":"2020-08-22T03:54:14.607086+0000","last_became_active":"2020-08-22T03:49:46.597567+0000","last_became_peered":"2020-08-22T03:49:46.597567+0000","last_unstale":"2020-08-22T03:54:14.607086+0000","last_undegraded":"2020-08-22T03:54:14.607086+0000","last_fullsized":"2020-08-22T03:54:14.607086+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":50,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"19'46","reported_seq":"57","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:14.867685+0000","last_change":"2020-08-22T03:54:14.867685+0000","last_active":"2020-08-22T03:54:14.867685+0000","last_peered":"2020-08-22T03:54:14.867685+0000","last_clean":"2020-08-22T03:54:14.867685+0000","last_became_active":"2020-08-22T03:49:31.550771+0000","last_became_peered":"2020-08-22T03:49:31.550771+0000","last_unstale":"2020-08-22T03:54:14.867685+0000","last_undegraded":"2020-08-22T03:54:14.867685+0000","last_fullsized":"2020-08-22T03:54:14.867685+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":46,"ondisk_log_size":46,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"19'47","reported_seq":"57","reported_epoch":"35","state":"active+clean+wait","last_fresh":"2020-08-22T03:54:14.733949+0000","last_change":"2020-08-22T03:54:14.733949+0000","last_active":"2020-08-22T03:54:14.733949+0000","last_peered":"2020-08-22T03:54:14.733949+0000","last_clean":"2020-08-22T03:54:14.733949+0000","last_became_active":"2020-08-22T03:54:14.733949+0000","last_became_peered":"2020-08-22T03:54:14.733949+0000","last_unstale":"2020-08-22T03:54:14.733949+0000","last_undegraded":"2020-08-22T03:54:14.733949+0000","last_fullsized":"2020-08-22T03:54:14.733949+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":47,"ondisk_log_size":47,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"19'46","reported_seq":"57","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:14.607068+0000","last_change":"2020-08-22T03:49:46.597549+0000","last_active":"2020-08-22T03:54:14.607068+0000","last_peered":"2020-08-22T03:54:14.607068+0000","last_clean":"2020-08-22T03:54:14.607068+0000","last_became_active":"2020-08-22T03:49:46.597549+0000","last_became_peered":"2020-08-22T03:49:46.597549+0000","last_unstale":"2020-08-22T03:54:14.607068+0000","last_undegraded":"2020-08-22T03:54:14.607068+0000","last_fullsized":"2020-08-22T03:54:14.607068+0000","mapping_epoch":24,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":27,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":46,"ondisk_log_size":46,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"18'33","reported_seq":"57","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:15.007309+0000","last_change":"2020-08-22T03:49:31.547763+0000","last_active":"2020-08-22T03:54:15.007309+0000","last_peered":"2020-08-22T03:54:15.007309+0000","last_clean":"2020-08-22T03:54:15.007309+0000","last_became_active":"2020-08-22T03:49:31.547763+0000","last_became_peered":"2020-08-22T03:49:31.547763+0000","last_unstale":"2020-08-22T03:54:15.007309+0000","last_undegraded":"2020-08-22T03:54:15.007309+0000","last_fullsized":"2020-08-22T03:54:15.007309+0000","mapping_epoch":24,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":25,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":33,"ondisk_log_size":33,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"19'59","reported_seq":"57","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:14.607051+0000","last_change":"2020-08-22T03:54:14.607051+0000","last_active":"2020-08-22T03:54:14.607051+0000","last_peered":"2020-08-22T03:54:14.607051+0000","last_clean":"2020-08-22T03:54:14.607051+0000","last_became_active":"2020-08-22T03:49:46.597450+0000","last_became_peered":"2020-08-22T03:49:46.597450+0000","last_unstale":"2020-08-22T03:54:14.607051+0000","last_undegraded":"2020-08-22T03:54:14.607051+0000","last_fullsized":"2020-08-22T03:54:14.607051+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":59,"ondisk_log_size":59,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"19'81","reported_seq":"57","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:14.607032+0000","last_change":"2020-08-22T03:54:14.607032+0000","last_active":"2020-08-22T03:54:14.607032+0000","last_peered":"2020-08-22T03:54:14.607032+0000","last_clean":"2020-08-22T03:54:14.607032+0000","last_became_active":"2020-08-22T03:49:46.597349+0000","last_became_peered":"2020-08-22T03:49:46.597349+0000","last_unstale":"2020-08-22T03:54:14.607032+0000","last_undegraded":"2020-08-22T03:54:14.607032+0000","last_fullsized":"2020-08-22T03:54:14.607032+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":81,"ondisk_log_size":81,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"19'56","reported_seq":"57","reported_epoch":"35","state":"active+clean+wait","last_fresh":"2020-08-22T03:54:14.733930+0000","last_change":"2020-08-22T03:54:14.733930+0000","last_active":"2020-08-22T03:54:14.733930+0000","last_peered":"2020-08-22T03:54:14.733930+0000","last_clean":"2020-08-22T03:54:14.733930+0000","last_became_active":"2020-08-22T03:54:14.733930+0000","last_became_peered":"2020-08-22T03:54:14.733930+0000","last_unstale":"2020-08-22T03:54:14.733930+0000","last_undegraded":"2020-08-22T03:54:14.733930+0000","last_fullsized":"2020-08-22T03:54:14.733930+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":56,"ondisk_log_size":56,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":"7","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:14.733853+0000","last_change":"2020-08-22T03:54:14.733853+0000","last_active":"2020-08-22T03:54:14.733853+0000","last_peered":"2020-08-22T03:54:14.733853+0000","last_clean":"2020-08-22T03:54:14.733853+0000","last_became_active":"2020-08-22T03:54:14.733853+0000","last_became_peered":"2020-08-22T03:54:14.733853+0000","last_unstale":"2020-08-22T03:54:14.733853+0000","last_undegraded":"2020-08-22T03:54:14.733853+0000","last_fullsized":"2020-08-22T03:54:14.733853+0000","mapping_epoch":34,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":35,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_clean_scrub_stamp":"2020-08-22T03:49:16.940536+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"19'41","reported_seq":"57","reported_epoch":"35","state":"active+clean+wait","last_fresh":"2020-08-22T03:54:14.733905+0000","last_change":"2020-08-22T03:54:14.733905+0000","last_active":"2020-08-22T03:54:14.733905+0000","last_peered":"2020-08-22T03:54:14.733905+0000","last_clean":"2020-08-22T03:54:14.733905+0000","last_became_active":"2020-08-22T03:54:14.733905+0000","last_became_peered":"2020-08-22T03:54:14.733905+0000","last_unstale":"2020-08-22T03:54:14.733905+0000","last_undegraded":"2020-08-22T03:54:14.733905+0000","last_fullsized":"2020-08-22T03:54:14.733905+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":41,"ondisk_log_size":41,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":"60","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:15.007212+0000","last_change":"2020-08-22T03:54:15.007212+0000","last_active":"2020-08-22T03:54:15.007212+0000","last_peered":"2020-08-22T03:54:15.007212+0000","last_clean":"2020-08-22T03:54:15.007212+0000","last_became_active":"2020-08-22T03:50:16.579309+0000","last_became_peered":"2020-08-22T03:50:16.579309+0000","last_unstale":"2020-08-22T03:54:15.007212+0000","last_undegraded":"2020-08-22T03:54:15.007212+0000","last_fullsized":"2020-08-22T03:54:15.007212+0000","mapping_epoch":34,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":35,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_clean_scrub_stamp":"2020-08-22T03:49:16.940536+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.3","version":"19'53","reported_seq":"57","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:14.867129+0000","last_change":"2020-08-22T03:49:31.550725+0000","last_active":"2020-08-22T03:54:14.867129+0000","last_peered":"2020-08-22T03:54:14.867129+0000","last_clean":"2020-08-22T03:54:14.867129+0000","last_became_active":"2020-08-22T03:49:31.550725+0000","last_became_peered":"2020-08-22T03:49:31.550725+0000","last_unstale":"2020-08-22T03:54:14.867129+0000","last_undegraded":"2020-08-22T03:54:14.867129+0000","last_fullsized":"2020-08-22T03:54:14.867129+0000","mapping_epoch":24,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":25,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":53,"ondisk_log_size":53,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":"60","reported_epoch":"35","state":"active+clean+wait","last_fresh":"2020-08-22T03:54:14.733805+0000","last_change":"2020-08-22T03:54:14.733805+0000","last_active":"2020-08-22T03:54:14.733805+0000","last_peered":"2020-08-22T03:54:14.733805+0000","last_clean":"2020-08-22T03:54:14.733805+0000","last_became_active":"2020-08-22T03:54:14.733805+0000","last_became_peered":"2020-08-22T03:54:14.733805+0000","last_unstale":"2020-08-22T03:54:14.733805+0000","last_undegraded":"2020-08-22T03:54:14.733805+0000","last_fullsized":"2020-08-22T03:54:14.733805+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_clean_scrub_stamp":"2020-08-22T03:49:16.940536+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"19'66","reported_seq":"57","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:15.007260+0000","last_change":"2020-08-22T03:54:15.007260+0000","last_active":"2020-08-22T03:54:15.007260+0000","last_peered":"2020-08-22T03:54:15.007260+0000","last_clean":"2020-08-22T03:54:15.007260+0000","last_became_active":"2020-08-22T03:49:31.547716+0000","last_became_peered":"2020-08-22T03:49:31.547716+0000","last_unstale":"2020-08-22T03:54:15.007260+0000","last_undegraded":"2020-08-22T03:54:15.007260+0000","last_fullsized":"2020-08-22T03:54:15.007260+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":66,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":"60","reported_epoch":"35","state":"active+clean+wait","last_fresh":"2020-08-22T03:54:14.733835+0000","last_change":"2020-08-22T03:54:14.733835+0000","last_active":"2020-08-22T03:54:14.733835+0000","last_peered":"2020-08-22T03:54:14.733835+0000","last_clean":"2020-08-22T03:54:14.733835+0000","last_became_active":"2020-08-22T03:49:46.597044+0000","last_became_peered":"2020-08-22T03:49:46.597044+0000","last_unstale":"2020-08-22T03:54:14.733835+0000","last_undegraded":"2020-08-22T03:54:14.733835+0000","last_fullsized":"2020-08-22T03:54:14.733835+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_clean_scrub_stamp":"2020-08-22T03:49:16.940536+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"9'1","reported_seq":"61","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:15.007189+0000","last_change":"2020-08-22T03:49:16.536195+0000","last_active":"2020-08-22T03:54:15.007189+0000","last_peered":"2020-08-22T03:54:15.007189+0000","last_clean":"2020-08-22T03:54:15.007189+0000","last_became_active":"2020-08-22T03:49:16.536195+0000","last_became_peered":"2020-08-22T03:49:16.536195+0000","last_unstale":"2020-08-22T03:54:15.007189+0000","last_undegraded":"2020-08-22T03:54:15.007189+0000","last_fullsized":"2020-08-22T03:54:15.007189+0000","mapping_epoch":24,"log_start":"0'0","ondisk_log_start":"0'0","created":7,"last_epoch_clean":25,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:10.478504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:10.478504+0000","last_clean_scrub_stamp":"2020-08-22T03:49:10.478504+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2","version":"19'50","reported_seq":"57","reported_epoch":"35","state":"active+clean+wait","last_fresh":"2020-08-22T03:54:14.733886+0000","last_change":"2020-08-22T03:54:14.733886+0000","last_active":"2020-08-22T03:54:14.733886+0000","last_peered":"2020-08-22T03:54:14.733886+0000","last_clean":"2020-08-22T03:54:14.733886+0000","last_became_active":"2020-08-22T03:49:46.597239+0000","last_became_peered":"2020-08-22T03:49:46.597239+0000","last_unstale":"2020-08-22T03:54:14.733886+0000","last_undegraded":"2020-08-22T03:54:14.733886+0000","last_fullsized":"2020-08-22T03:54:14.733886+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":50,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":"61","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:14.607004+0000","last_change":"2020-08-22T03:54:14.607004+0000","last_active":"2020-08-22T03:54:14.607004+0000","last_peered":"2020-08-22T03:54:14.607004+0000","last_clean":"2020-08-22T03:54:14.607004+0000","last_became_active":"2020-08-22T03:49:46.596831+0000","last_became_peered":"2020-08-22T03:49:46.596831+0000","last_unstale":"2020-08-22T03:54:14.607004+0000","last_undegraded":"2020-08-22T03:54:14.607004+0000","last_fullsized":"2020-08-22T03:54:14.607004+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_clean_scrub_stamp":"2020-08-22T03:49:16.940536+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"19'43","reported_seq":"57","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:15.007276+0000","last_change":"2020-08-22T03:54:15.007276+0000","last_active":"2020-08-22T03:54:15.007276+0000","last_peered":"2020-08-22T03:54:15.007276+0000","last_clean":"2020-08-22T03:54:15.007276+0000","last_became_active":"2020-08-22T03:49:31.547732+0000","last_became_peered":"2020-08-22T03:49:31.547732+0000","last_unstale":"2020-08-22T03:54:15.007276+0000","last_undegraded":"2020-08-22T03:54:15.007276+0000","last_fullsized":"2020-08-22T03:54:15.007276+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":43,"ondisk_log_size":43,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":"60","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:15.007245+0000","last_change":"2020-08-22T03:49:20.323711+0000","last_active":"2020-08-22T03:54:15.007245+0000","last_peered":"2020-08-22T03:54:15.007245+0000","last_clean":"2020-08-22T03:54:15.007245+0000","last_became_active":"2020-08-22T03:49:20.323711+0000","last_became_peered":"2020-08-22T03:49:20.323711+0000","last_unstale":"2020-08-22T03:54:15.007245+0000","last_undegraded":"2020-08-22T03:54:15.007245+0000","last_fullsized":"2020-08-22T03:54:15.007245+0000","mapping_epoch":24,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":25,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_clean_scrub_stamp":"2020-08-22T03:49:16.940536+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"19'56","reported_seq":"57","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:14.867146+0000","last_change":"2020-08-22T03:49:31.550741+0000","last_active":"2020-08-22T03:54:14.867146+0000","last_peered":"2020-08-22T03:54:14.867146+0000","last_clean":"2020-08-22T03:54:14.867146+0000","last_became_active":"2020-08-22T03:49:31.550741+0000","last_became_peered":"2020-08-22T03:49:31.550741+0000","last_unstale":"2020-08-22T03:54:14.867146+0000","last_undegraded":"2020-08-22T03:54:14.867146+0000","last_fullsized":"2020-08-22T03:54:14.867146+0000","mapping_epoch":24,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":25,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":56,"ondisk_log_size":56,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"19'57","reported_seq":"57","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:15.007293+0000","last_change":"2020-08-22T03:54:15.007293+0000","last_active":"2020-08-22T03:54:15.007293+0000","last_peered":"2020-08-22T03:54:15.007293+0000","last_clean":"2020-08-22T03:54:15.007293+0000","last_became_active":"2020-08-22T03:49:31.547748+0000","last_became_peered":"2020-08-22T03:49:31.547748+0000","last_unstale":"2020-08-22T03:54:15.007293+0000","last_undegraded":"2020-08-22T03:54:15.007293+0000","last_fullsized":"2020-08-22T03:54:15.007293+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":57,"ondisk_log_size":57,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":"60","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:14.867108+0000","last_change":"2020-08-22T03:49:20.178704+0000","last_active":"2020-08-22T03:54:14.867108+0000","last_peered":"2020-08-22T03:54:14.867108+0000","last_clean":"2020-08-22T03:54:14.867108+0000","last_became_active":"2020-08-22T03:49:20.178704+0000","last_became_peered":"2020-08-22T03:49:20.178704+0000","last_unstale":"2020-08-22T03:54:14.867108+0000","last_undegraded":"2020-08-22T03:54:14.867108+0000","last_fullsized":"2020-08-22T03:54:14.867108+0000","mapping_epoch":24,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":25,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_clean_scrub_stamp":"2020-08-22T03:49:16.940536+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":"60","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:14.733870+0000","last_change":"2020-08-22T03:54:14.733870+0000","last_active":"2020-08-22T03:54:14.733870+0000","last_peered":"2020-08-22T03:54:14.733870+0000","last_clean":"2020-08-22T03:54:14.733870+0000","last_became_active":"2020-08-22T03:50:16.611490+0000","last_became_peered":"2020-08-22T03:50:16.611490+0000","last_unstale":"2020-08-22T03:54:14.733870+0000","last_undegraded":"2020-08-22T03:54:14.733870+0000","last_fullsized":"2020-08-22T03:54:14.733870+0000","mapping_epoch":34,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":35,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_clean_scrub_stamp":"2020-08-22T03:49:16.940536+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.6","version":"19'42","reported_seq":"57","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:14.867666+0000","last_change":"2020-08-22T03:54:14.867666+0000","last_active":"2020-08-22T03:54:14.867666+0000","last_peered":"2020-08-22T03:54:14.867666+0000","last_clean":"2020-08-22T03:54:14.867666+0000","last_became_active":"2020-08-22T03:49:31.550756+0000","last_became_peered":"2020-08-22T03:49:31.550756+0000","last_unstale":"2020-08-22T03:54:14.867666+0000","last_undegraded":"2020-08-22T03:54:14.867666+0000","last_fullsized":"2020-08-22T03:54:14.867666+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":42,"ondisk_log_size":42,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":826,"ondisk_log_size":826,"up":48,"acting":48,"num_store_stats":0},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1,"ondisk_log_size":1,"up":3,"acting":3,"num_store_stats":0},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":0}],"osd_stats":[{"osd":3,"up_from":24,"seq":103079215167,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":24,"seq":103079215167,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":8,"seq":34359738431,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":33,"seq":141733920831,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}]}} 2020-08-22T03:56:58.464 WARNING:scrape:Ignoring malparsed backtrace: File "src/gevent/greenlet.py", line 709, in gevent._greenlet.Greenlet.get , File "src/gevent/greenlet.py", line 317, in gevent._greenlet.Greenlet._raise_exception , File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/lib/python3.6/site-packages/gevent/_compat.py", line 47, in reraise 2020-08-22T03:56:58.465 WARNING:scrape:Ignoring malparsed backtrace: teuthology.exceptions.CommandCrashedError: Command crashed: 'CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --max-ops 400000 --objects 1024 --max-in-flight 64 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 600 --op read 100 --op write 50 --op delete 10 --op write_excl 50 --pool unique_pool_0' , 2020-08-22T03:54:44.981 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits... , 2020-08-22T03:54:44.982 INFO:teuthology.orchestra.run.smithi092:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits 2020-08-22T03:56:58.466 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:54:47.212 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-test x86_64 2:16.0.0-4564.ge64d8132f64.el8 @ceph 204 M , 2020-08-22T03:54:47.212 INFO:teuthology.orchestra.run.smithi092.stdout:Removing unused dependencies: , 2020-08-22T03:54:47.212 INFO:teuthology.orchestra.run.smithi092.stdout: jq x86_64 1.5-12.el8 @CentOS-AppStream 359 k 2020-08-22T03:56:58.467 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:54:49.701 INFO:teuthology.orchestra.run.smithi092.stdout:Freed space: 97 M , 2020-08-22T03:54:49.702 INFO:teuthology.orchestra.run.smithi092.stdout:Running transaction check , 2020-08-22T03:54:49.703 INFO:teuthology.orchestra.run.smithi092.stdout:Transaction check succeeded. 2020-08-22T03:56:58.469 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:54:52.586 INFO:teuthology.orchestra.run.smithi183.stdout: libgfortran x86_648.3.1-4.5.el8 @CentOS-Base 2.5 M , 2020-08-22T03:54:52.586 INFO:teuthology.orchestra.run.smithi183.stdout: liboath x86_642.6.2-3.el8 @epel 125 k , 2020-08-22T03:54:52.586 INFO:teuthology.orchestra.run.smithi183.stdout: libquadmath x86_648.3.1-4.5.el8 @CentOS-Base 299 k 2020-08-22T03:56:58.471 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:54:52.664 INFO:teuthology.orchestra.run.smithi092.stdout: libgfortran x86_648.3.1-4.5.el8 @CentOS-Base 2.5 M , 2020-08-22T03:54:52.664 INFO:teuthology.orchestra.run.smithi092.stdout: liboath x86_642.6.2-3.el8 @epel 125 k , 2020-08-22T03:54:52.664 INFO:teuthology.orchestra.run.smithi092.stdout: libquadmath x86_648.3.1-4.5.el8 @CentOS-Base 299 k 2020-08-22T03:56:58.472 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:54:54.568 INFO:teuthology.orchestra.run.smithi092.stdout: Erasing : ceph-mgr-dashboard-2:16.0.0-4564.ge64d8132f64.el8. 5/85 , 2020-08-22T03:54:54.585 INFO:teuthology.orchestra.run.smithi183.stdout: Erasing : python3-pecan-1.3.2-9.el8.noarch 4/85 , 2020-08-22T03:54:54.585 INFO:teuthology.orchestra.run.smithi183.stdout: Erasing : ceph-mgr-dashboard-2:16.0.0-4564.ge64d8132f64.el8. 5/85 2020-08-22T03:56:58.473 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:55:02.759 INFO:teuthology.orchestra.run.smithi183.stdout: Erasing : python3-certifi-2018.10.15-7.el8.noarch 48/85 , 2020-08-22T03:55:02.807 INFO:teuthology.orchestra.run.smithi092.stdout: Erasing : ceph-grafana-dashboards-2:16.0.0-4564.ge64d8132f64 50/85 , 2020-08-22T03:55:02.923 INFO:teuthology.orchestra.run.smithi092.stdout: Erasing : ceph-prometheus-alerts-2:16.0.0-4564.ge64d8132f64. 51/85 2020-08-22T03:56:58.475 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:55:22.603 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: libquadmath-8.3.1-4.5.el8.x86_64 77/85 , 2020-08-22T03:55:22.619 INFO:teuthology.orchestra.run.smithi183.stdout: Erasing : python3-rados-2:16.0.0-4564.ge64d8132f64.el8.x86_6 76/85 , 2020-08-22T03:55:22.627 INFO:teuthology.orchestra.run.smithi183.stdout: Running scriptlet: libquadmath-8.3.1-4.5.el8.x86_64 77/85 2020-08-22T03:56:58.476 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:55:24.187 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-urllib3-1.24.2-2.el8.noarch 77/85 , 2020-08-22T03:55:24.188 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-waitress-1.2.1-2.el8.1.noarch 78/85 , 2020-08-22T03:55:24.189 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-webencodings-0.5.1-6.el8.noarch 79/85 2020-08-22T03:56:58.476 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:55:25.495 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-grafana-dashboards-2:16.0.0-4564.ge64d8132f64.el8.noarch , 2020-08-22T03:55:25.495 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-mgr-2:16.0.0-4564.ge64d8132f64.el8.x86_64 , 2020-08-22T03:55:25.495 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-mgr-cephadm-2:16.0.0-4564.ge64d8132f64.el8.noarch 2020-08-22T03:56:58.477 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:55:26.146 INFO:teuthology.orchestra.run.smithi183.stdout: liboath-2.6.2-3.el8.x86_64 , 2020-08-22T03:55:26.146 INFO:teuthology.orchestra.run.smithi183.stdout: libquadmath-8.3.1-4.5.el8.x86_64 , 2020-08-22T03:55:26.146 INFO:teuthology.orchestra.run.smithi183.stdout: librabbitmq-0.9.0-1.el8.x86_64 2020-08-22T03:56:58.478 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:55:26.818 INFO:teuthology.orchestra.run.smithi183.stdout: , 2020-08-22T03:55:26.818 INFO:teuthology.orchestra.run.smithi183.stdout:Freed space: 185 k , 2020-08-22T03:55:26.819 INFO:teuthology.orchestra.run.smithi183.stdout:Running transaction check 2020-08-22T03:56:58.479 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:55:31.092 INFO:teuthology.orchestra.run.smithi092.stdout:Nothing to do. , 2020-08-22T03:55:31.093 INFO:teuthology.orchestra.run.smithi092.stdout:Complete! , 2020-08-22T03:55:31.156 INFO:teuthology.orchestra.run.smithi183.stdout:No match for argument: ceph-mgr-dashboard 2020-08-22T03:56:58.479 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:55:34.096 INFO:teuthology.orchestra.run.smithi183.stdout: Verifying : fuse-common-3.2.1-12.el8.x86_64 3/3 , 2020-08-22T03:55:34.096 INFO:teuthology.orchestra.run.smithi183.stdout: , 2020-08-22T03:55:34.096 INFO:teuthology.orchestra.run.smithi183.stdout:Removed: 2020-08-22T03:56:58.480 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:55:37.380 INFO:teuthology.orchestra.run.smithi092.stdout:Dependencies resolved. , 2020-08-22T03:55:37.381 INFO:teuthology.orchestra.run.smithi092.stdout:================================================================================ , 2020-08-22T03:55:37.382 INFO:teuthology.orchestra.run.smithi092.stdout: Package Arch Version Repository Size 2020-08-22T03:56:58.481 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:55:39.358 INFO:teuthology.orchestra.run.smithi183.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 4/7 , 2020-08-22T03:55:39.358 INFO:teuthology.orchestra.run.smithi183.stdout: Verifying : qemu-kvm-block-rbd-15:2.12.0-88.module_el8.1.0+297+d 5/7 , 2020-08-22T03:55:39.358 INFO:teuthology.orchestra.run.smithi183.stdout: Verifying : rbd-fuse-2:16.0.0-4564.ge64d8132f64.el8.x86_64 6/7 2020-08-22T03:56:58.481 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:55:43.855 INFO:teuthology.orchestra.run.smithi092:> sudo rm -f /etc/yum.repos.d/ceph.repo , 2020-08-22T03:55:43.921 INFO:teuthology.orchestra.run.smithi092:> sudo yum clean expire-cache , 2020-08-22T03:55:44.009 INFO:teuthology.orchestra.run.smithi183.stdout:51 files removed 2020-08-22T03:56:58.482 WARNING:scrape:Ignoring malparsed backtrace: next(self.gen) , File "/home/teuthworker/src/github.com_tchaikov_ceph_wip-crimson-recovery-qa/qa/tasks/rados.py", line 277, in task , running.get() 2020-08-22T03:56:58.515 WARNING:scrape:Ignoring malparsed backtrace: ceph version 16.0.0-4564-ge64d8132f64 (e64d8132f648167078f5841ad65b42669a68d2cb) pacific (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe0) [0x7ff89e69a39e] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x67d) [0x55dea91d2add] 2020-08-22T03:56:58.518 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:50:14.250 INFO:teuthology.orchestra.run.smithi183.stdout: , 2020-08-22T03:50:14.250 INFO:teuthology.orchestra.run.smithi183.stdout:{"epoch":32,"fsid":"9972e83e-5caf-455d-81f6-10e7c5bcb3e2","created":"2020-08-22T03:49:05.967889+0000","modified":"2020-08-22T03:50:11.943469+0000","last_up_change":"2020-08-22T03:49:42.013320+0000","last_in_change":"2020-08-22T03:49:11.484438+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-22T03:49:09.511038+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-22T03:49:16.052382+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-22T03:49:28.065828+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"e0d320b8-4993-4b1a-b741-ab771add082d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6800","nonce":25504}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6801","nonce":25504}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6806","nonce":25504}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6807","nonce":25504}]},"public_addr":"172.21.15.183:6800/25504","cluster_addr":"172.21.15.183:6801/25504","heartbeat_back_addr":"172.21.15.183:6806/25504","heartbeat_front_addr":"172.21.15.183:6807/25504","state":["exists","up"]},{"osd":1,"uuid":"51144378-a6ca-485c-8a15-20c001a1ed60","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":18,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6802","nonce":25547}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6803","nonce":25547}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6804","nonce":25547}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6805","nonce":25547}]},"public_addr":"172.21.15.183:6802/25547","cluster_addr":"172.21.15.183:6803/25547","heartbeat_back_addr":"172.21.15.183:6804/25547","heartbeat_front_addr":"172.21.15.183:6805/25547","state":["exists"]},{"osd":2,"uuid":"9f4f44e0-4c21-49c7-83c6-9fd9c902a564","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":23,"up_from":24,"up_thru":24,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6802","nonce":25353}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6803","nonce":25353}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6806","nonce":25353}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6807","nonce":25353}]},"public_addr":"172.21.15.92:6802/25353","cluster_addr":"172.21.15.92:6803/25353","heartbeat_back_addr":"172.21.15.92:6806/25353","heartbeat_front_addr":"172.21.15.92:6807/25353","state":["exists","up"]},{"osd":3,"uuid":"39fb66cd-8671-4a52-a6bd-6376d5ff4f5b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":23,"up_from":24,"up_thru":31,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6804","nonce":25397}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6805","nonce":25397}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6808","nonce":25397}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6809","nonce":25397}]},"public_addr":"172.21.15.92:6804/25397","cluster_addr":"172.21.15.92:6805/25397","heartbeat_back_addr":"172.21.15.92:6808/25397","heartbeat_front_addr":"172.21.15.92:6809/25397","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-22T03:49:41.010632+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"2020-08-22T03:49:41.010632+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-22T03:49:41.010632+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.2","mappings":[{"from":0,"to":3}]},{"pgid":"2.4","mappings":[{"from":2,"to":1}]},{"pgid":"2.7","mappings":[{"from":2,"to":3}]}],"pg_temp":[{"pgid":"2.2","osds":[3,2,0]},{"pgid":"2.4","osds":[3,0,2]},{"pgid":"2.7","osds":[0,3,2]}],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-22T03:50:15.942+0000 7f343b7fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.0 v2:172.21.15.183:6800/25504 for 24sec e20 v20) 2020-08-22T03:56:58.519 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:51:43.036 INFO:teuthology.orchestra.run.smithi183.stdout:{"epoch":32,"fsid":"9972e83e-5caf-455d-81f6-10e7c5bcb3e2","created":"2020-08-22T03:49:05.967889+0000","modified":"2020-08-22T03:50:11.943469+0000","last_up_change":"2020-08-22T03:49:42.013320+0000","last_in_change":"2020-08-22T03:49:11.484438+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-22T03:49:09.511038+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-22T03:49:16.052382+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-22T03:49:28.065828+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"e0d320b8-4993-4b1a-b741-ab771add082d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6800","nonce":25504}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6801","nonce":25504}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6806","nonce":25504}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6807","nonce":25504}]},"public_addr":"172.21.15.183:6800/25504","cluster_addr":"172.21.15.183:6801/25504","heartbeat_back_addr":"172.21.15.183:6806/25504","heartbeat_front_addr":"172.21.15.183:6807/25504","state":["exists","up"]},{"osd":1,"uuid":"51144378-a6ca-485c-8a15-20c001a1ed60","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":18,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6802","nonce":25547}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6803","nonce":25547}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6804","nonce":25547}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6805","nonce":25547}]},"public_addr":"172.21.15.183:6802/25547","cluster_addr":"172.21.15.183:6803/25547","heartbeat_back_addr":"172.21.15.183:6804/25547","heartbeat_front_addr":"172.21.15.183:6805/25547","state":["exists"]},{"osd":2,"uuid":"9f4f44e0-4c21-49c7-83c6-9fd9c902a564","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":23,"up_from":24,"up_thru":24,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6802","nonce":25353}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6803","nonce":25353}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6806","nonce":25353}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6807","nonce":25353}]},"public_addr":"172.21.15.92:6802/25353","cluster_addr":"172.21.15.92:6803/25353","heartbeat_back_addr":"172.21.15.92:6806/25353","heartbeat_front_addr":"172.21.15.92:6807/25353","state":["exists","up"]},{"osd":3,"uuid":"39fb66cd-8671-4a52-a6bd-6376d5ff4f5b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":23,"up_from":24,"up_thru":31,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6804","nonce":25397}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6805","nonce":25397}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6808","nonce":25397}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6809","nonce":25397}]},"public_addr":"172.21.15.92:6804/25397","cluster_addr":"172.21.15.92:6805/25397","heartbeat_back_addr":"172.21.15.92:6808/25397","heartbeat_front_addr":"172.21.15.92:6809/25397","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-22T03:49:41.010632+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"2020-08-22T03:49:41.010632+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-22T03:49:41.010632+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.2","mappings":[{"from":0,"to":3}]},{"pgid":"2.4","mappings":[{"from":2,"to":1}]},{"pgid":"2.7","mappings":[{"from":2,"to":3}]}],"pg_temp":[{"pgid":"2.2","osds":[3,2,0]},{"pgid":"2.4","osds":[3,0,2]},{"pgid":"2.7","osds":[0,3,2]}],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-22T03:51:45.951+0000 7f343b7fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.0 v2:172.21.15.183:6800/25504 for 24sec e20 v20) , 2020-08-22T03:51:46.046 INFO:teuthology.orchestra.run.smithi183:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json 2020-08-22T03:56:58.521 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:53:11.768 INFO:teuthology.orchestra.run.smithi183.stdout:{"epoch":32,"fsid":"9972e83e-5caf-455d-81f6-10e7c5bcb3e2","created":"2020-08-22T03:49:05.967889+0000","modified":"2020-08-22T03:50:11.943469+0000","last_up_change":"2020-08-22T03:49:42.013320+0000","last_in_change":"2020-08-22T03:49:11.484438+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-22T03:49:09.511038+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-22T03:49:16.052382+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-22T03:49:28.065828+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"e0d320b8-4993-4b1a-b741-ab771add082d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6800","nonce":25504}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6801","nonce":25504}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6806","nonce":25504}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6807","nonce":25504}]},"public_addr":"172.21.15.183:6800/25504","cluster_addr":"172.21.15.183:6801/25504","heartbeat_back_addr":"172.21.15.183:6806/25504","heartbeat_front_addr":"172.21.15.183:6807/25504","state":["exists","up"]},{"osd":1,"uuid":"51144378-a6ca-485c-8a15-20c001a1ed60","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":18,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6802","nonce":25547}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6803","nonce":25547}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6804","nonce":25547}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6805","nonce":25547}]},"public_addr":"172.21.15.183:6802/25547","cluster_addr":"172.21.15.183:6803/25547","heartbeat_back_addr":"172.21.15.183:6804/25547","heartbeat_front_addr":"172.21.15.183:6805/25547","state":["exists"]},{"osd":2,"uuid":"9f4f44e0-4c21-49c7-83c6-9fd9c902a564","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":23,"up_from":24,"up_thru":24,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6802","nonce":25353}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6803","nonce":25353}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6806","nonce":25353}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6807","nonce":25353}]},"public_addr":"172.21.15.92:6802/25353","cluster_addr":"172.21.15.92:6803/25353","heartbeat_back_addr":"172.21.15.92:6806/25353","heartbeat_front_addr":"172.21.15.92:6807/25353","state":["exists","up"]},{"osd":3,"uuid":"39fb66cd-8671-4a52-a6bd-6376d5ff4f5b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":23,"up_from":24,"up_thru":31,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6804","nonce":25397}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6805","nonce":25397}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6808","nonce":25397}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6809","nonce":25397}]},"public_addr":"172.21.15.92:6804/25397","cluster_addr":"172.21.15.92:6805/25397","heartbeat_back_addr":"172.21.15.92:6808/25397","heartbeat_front_addr":"172.21.15.92:6809/25397","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-22T03:49:41.010632+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"2020-08-22T03:49:41.010632+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-22T03:49:41.010632+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.2","mappings":[{"from":0,"to":3}]},{"pgid":"2.4","mappings":[{"from":2,"to":1}]},{"pgid":"2.7","mappings":[{"from":2,"to":3}]}],"pg_temp":[{"pgid":"2.2","osds":[3,2,0]},{"pgid":"2.4","osds":[3,0,2]},{"pgid":"2.7","osds":[0,3,2]}],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-22T03:53:14.780 INFO:teuthology.orchestra.run.smithi183:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-22T03:53:15.052 INFO:teuthology.orchestra.run.smithi183.stdout: 2020-08-22T03:56:58.524 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:54:17.718 INFO:teuthology.orchestra.run.smithi183.stdout: , dumped all , 2020-08-22T03:54:17.732 INFO:teuthology.orchestra.run.smithi183.stdout:{"pg_ready":true,"pg_map":{"version":183,"stamp":"2020-08-22T03:54:17.521444+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":827,"ondisk_log_size":827,"up":75,"acting":75,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":0,"num_osds":4,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"3.f","version":"19'50","reported_seq":"58","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:14.607086+0000","last_change":"2020-08-22T03:54:14.607086+0000","last_active":"2020-08-22T03:54:14.607086+0000","last_peered":"2020-08-22T03:54:14.607086+0000","last_clean":"2020-08-22T03:54:14.607086+0000","last_became_active":"2020-08-22T03:49:46.597567+0000","last_became_peered":"2020-08-22T03:49:46.597567+0000","last_unstale":"2020-08-22T03:54:14.607086+0000","last_undegraded":"2020-08-22T03:54:14.607086+0000","last_fullsized":"2020-08-22T03:54:14.607086+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":50,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"19'46","reported_seq":"57","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:14.867685+0000","last_change":"2020-08-22T03:54:14.867685+0000","last_active":"2020-08-22T03:54:14.867685+0000","last_peered":"2020-08-22T03:54:14.867685+0000","last_clean":"2020-08-22T03:54:14.867685+0000","last_became_active":"2020-08-22T03:49:31.550771+0000","last_became_peered":"2020-08-22T03:49:31.550771+0000","last_unstale":"2020-08-22T03:54:14.867685+0000","last_undegraded":"2020-08-22T03:54:14.867685+0000","last_fullsized":"2020-08-22T03:54:14.867685+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":46,"ondisk_log_size":46,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"19'47","reported_seq":"57","reported_epoch":"35","state":"active+clean+wait","last_fresh":"2020-08-22T03:54:14.733949+0000","last_change":"2020-08-22T03:54:14.733949+0000","last_active":"2020-08-22T03:54:14.733949+0000","last_peered":"2020-08-22T03:54:14.733949+0000","last_clean":"2020-08-22T03:54:14.733949+0000","last_became_active":"2020-08-22T03:54:14.733949+0000","last_became_peered":"2020-08-22T03:54:14.733949+0000","last_unstale":"2020-08-22T03:54:14.733949+0000","last_undegraded":"2020-08-22T03:54:14.733949+0000","last_fullsized":"2020-08-22T03:54:14.733949+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":47,"ondisk_log_size":47,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"19'46","reported_seq":"57","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:14.607068+0000","last_change":"2020-08-22T03:49:46.597549+0000","last_active":"2020-08-22T03:54:14.607068+0000","last_peered":"2020-08-22T03:54:14.607068+0000","last_clean":"2020-08-22T03:54:14.607068+0000","last_became_active":"2020-08-22T03:49:46.597549+0000","last_became_peered":"2020-08-22T03:49:46.597549+0000","last_unstale":"2020-08-22T03:54:14.607068+0000","last_undegraded":"2020-08-22T03:54:14.607068+0000","last_fullsized":"2020-08-22T03:54:14.607068+0000","mapping_epoch":24,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":27,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":46,"ondisk_log_size":46,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"18'33","reported_seq":"57","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:15.007309+0000","last_change":"2020-08-22T03:49:31.547763+0000","last_active":"2020-08-22T03:54:15.007309+0000","last_peered":"2020-08-22T03:54:15.007309+0000","last_clean":"2020-08-22T03:54:15.007309+0000","last_became_active":"2020-08-22T03:49:31.547763+0000","last_became_peered":"2020-08-22T03:49:31.547763+0000","last_unstale":"2020-08-22T03:54:15.007309+0000","last_undegraded":"2020-08-22T03:54:15.007309+0000","last_fullsized":"2020-08-22T03:54:15.007309+0000","mapping_epoch":24,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":25,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":33,"ondisk_log_size":33,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"19'59","reported_seq":"57","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:14.607051+0000","last_change":"2020-08-22T03:54:14.607051+0000","last_active":"2020-08-22T03:54:14.607051+0000","last_peered":"2020-08-22T03:54:14.607051+0000","last_clean":"2020-08-22T03:54:14.607051+0000","last_became_active":"2020-08-22T03:49:46.597450+0000","last_became_peered":"2020-08-22T03:49:46.597450+0000","last_unstale":"2020-08-22T03:54:14.607051+0000","last_undegraded":"2020-08-22T03:54:14.607051+0000","last_fullsized":"2020-08-22T03:54:14.607051+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":59,"ondisk_log_size":59,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"19'81","reported_seq":"57","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:14.607032+0000","last_change":"2020-08-22T03:54:14.607032+0000","last_active":"2020-08-22T03:54:14.607032+0000","last_peered":"2020-08-22T03:54:14.607032+0000","last_clean":"2020-08-22T03:54:14.607032+0000","last_became_active":"2020-08-22T03:49:46.597349+0000","last_became_peered":"2020-08-22T03:49:46.597349+0000","last_unstale":"2020-08-22T03:54:14.607032+0000","last_undegraded":"2020-08-22T03:54:14.607032+0000","last_fullsized":"2020-08-22T03:54:14.607032+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":81,"ondisk_log_size":81,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"19'56","reported_seq":"57","reported_epoch":"35","state":"active+clean+wait","last_fresh":"2020-08-22T03:54:14.733930+0000","last_change":"2020-08-22T03:54:14.733930+0000","last_active":"2020-08-22T03:54:14.733930+0000","last_peered":"2020-08-22T03:54:14.733930+0000","last_clean":"2020-08-22T03:54:14.733930+0000","last_became_active":"2020-08-22T03:54:14.733930+0000","last_became_peered":"2020-08-22T03:54:14.733930+0000","last_unstale":"2020-08-22T03:54:14.733930+0000","last_undegraded":"2020-08-22T03:54:14.733930+0000","last_fullsized":"2020-08-22T03:54:14.733930+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":56,"ondisk_log_size":56,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":"7","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:14.733853+0000","last_change":"2020-08-22T03:54:14.733853+0000","last_active":"2020-08-22T03:54:14.733853+0000","last_peered":"2020-08-22T03:54:14.733853+0000","last_clean":"2020-08-22T03:54:14.733853+0000","last_became_active":"2020-08-22T03:54:14.733853+0000","last_became_peered":"2020-08-22T03:54:14.733853+0000","last_unstale":"2020-08-22T03:54:14.733853+0000","last_undegraded":"2020-08-22T03:54:14.733853+0000","last_fullsized":"2020-08-22T03:54:14.733853+0000","mapping_epoch":34,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":35,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_clean_scrub_stamp":"2020-08-22T03:49:16.940536+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"19'41","reported_seq":"57","reported_epoch":"35","state":"active+clean+wait","last_fresh":"2020-08-22T03:54:14.733905+0000","last_change":"2020-08-22T03:54:14.733905+0000","last_active":"2020-08-22T03:54:14.733905+0000","last_peered":"2020-08-22T03:54:14.733905+0000","last_clean":"2020-08-22T03:54:14.733905+0000","last_became_active":"2020-08-22T03:54:14.733905+0000","last_became_peered":"2020-08-22T03:54:14.733905+0000","last_unstale":"2020-08-22T03:54:14.733905+0000","last_undegraded":"2020-08-22T03:54:14.733905+0000","last_fullsized":"2020-08-22T03:54:14.733905+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":41,"ondisk_log_size":41,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":"60","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:15.007212+0000","last_change":"2020-08-22T03:54:15.007212+0000","last_active":"2020-08-22T03:54:15.007212+0000","last_peered":"2020-08-22T03:54:15.007212+0000","last_clean":"2020-08-22T03:54:15.007212+0000","last_became_active":"2020-08-22T03:50:16.579309+0000","last_became_peered":"2020-08-22T03:50:16.579309+0000","last_unstale":"2020-08-22T03:54:15.007212+0000","last_undegraded":"2020-08-22T03:54:15.007212+0000","last_fullsized":"2020-08-22T03:54:15.007212+0000","mapping_epoch":34,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":35,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_clean_scrub_stamp":"2020-08-22T03:49:16.940536+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.3","version":"19'53","reported_seq":"57","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:14.867129+0000","last_change":"2020-08-22T03:49:31.550725+0000","last_active":"2020-08-22T03:54:14.867129+0000","last_peered":"2020-08-22T03:54:14.867129+0000","last_clean":"2020-08-22T03:54:14.867129+0000","last_became_active":"2020-08-22T03:49:31.550725+0000","last_became_peered":"2020-08-22T03:49:31.550725+0000","last_unstale":"2020-08-22T03:54:14.867129+0000","last_undegraded":"2020-08-22T03:54:14.867129+0000","last_fullsized":"2020-08-22T03:54:14.867129+0000","mapping_epoch":24,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":25,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":53,"ondisk_log_size":53,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":"60","reported_epoch":"35","state":"active+clean+wait","last_fresh":"2020-08-22T03:54:14.733805+0000","last_change":"2020-08-22T03:54:14.733805+0000","last_active":"2020-08-22T03:54:14.733805+0000","last_peered":"2020-08-22T03:54:14.733805+0000","last_clean":"2020-08-22T03:54:14.733805+0000","last_became_active":"2020-08-22T03:54:14.733805+0000","last_became_peered":"2020-08-22T03:54:14.733805+0000","last_unstale":"2020-08-22T03:54:14.733805+0000","last_undegraded":"2020-08-22T03:54:14.733805+0000","last_fullsized":"2020-08-22T03:54:14.733805+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_clean_scrub_stamp":"2020-08-22T03:49:16.940536+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"19'66","reported_seq":"57","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:15.007260+0000","last_change":"2020-08-22T03:54:15.007260+0000","last_active":"2020-08-22T03:54:15.007260+0000","last_peered":"2020-08-22T03:54:15.007260+0000","last_clean":"2020-08-22T03:54:15.007260+0000","last_became_active":"2020-08-22T03:49:31.547716+0000","last_became_peered":"2020-08-22T03:49:31.547716+0000","last_unstale":"2020-08-22T03:54:15.007260+0000","last_undegraded":"2020-08-22T03:54:15.007260+0000","last_fullsized":"2020-08-22T03:54:15.007260+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":66,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":"60","reported_epoch":"35","state":"active+clean+wait","last_fresh":"2020-08-22T03:54:14.733835+0000","last_change":"2020-08-22T03:54:14.733835+0000","last_active":"2020-08-22T03:54:14.733835+0000","last_peered":"2020-08-22T03:54:14.733835+0000","last_clean":"2020-08-22T03:54:14.733835+0000","last_became_active":"2020-08-22T03:49:46.597044+0000","last_became_peered":"2020-08-22T03:49:46.597044+0000","last_unstale":"2020-08-22T03:54:14.733835+0000","last_undegraded":"2020-08-22T03:54:14.733835+0000","last_fullsized":"2020-08-22T03:54:14.733835+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_clean_scrub_stamp":"2020-08-22T03:49:16.940536+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"9'1","reported_seq":"61","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:15.007189+0000","last_change":"2020-08-22T03:49:16.536195+0000","last_active":"2020-08-22T03:54:15.007189+0000","last_peered":"2020-08-22T03:54:15.007189+0000","last_clean":"2020-08-22T03:54:15.007189+0000","last_became_active":"2020-08-22T03:49:16.536195+0000","last_became_peered":"2020-08-22T03:49:16.536195+0000","last_unstale":"2020-08-22T03:54:15.007189+0000","last_undegraded":"2020-08-22T03:54:15.007189+0000","last_fullsized":"2020-08-22T03:54:15.007189+0000","mapping_epoch":24,"log_start":"0'0","ondisk_log_start":"0'0","created":7,"last_epoch_clean":25,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:10.478504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:10.478504+0000","last_clean_scrub_stamp":"2020-08-22T03:49:10.478504+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2","version":"19'50","reported_seq":"57","reported_epoch":"35","state":"active+clean+wait","last_fresh":"2020-08-22T03:54:14.733886+0000","last_change":"2020-08-22T03:54:14.733886+0000","last_active":"2020-08-22T03:54:14.733886+0000","last_peered":"2020-08-22T03:54:14.733886+0000","last_clean":"2020-08-22T03:54:14.733886+0000","last_became_active":"2020-08-22T03:49:46.597239+0000","last_became_peered":"2020-08-22T03:49:46.597239+0000","last_unstale":"2020-08-22T03:54:14.733886+0000","last_undegraded":"2020-08-22T03:54:14.733886+0000","last_fullsized":"2020-08-22T03:54:14.733886+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":50,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":"61","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:14.607004+0000","last_change":"2020-08-22T03:54:14.607004+0000","last_active":"2020-08-22T03:54:14.607004+0000","last_peered":"2020-08-22T03:54:14.607004+0000","last_clean":"2020-08-22T03:54:14.607004+0000","last_became_active":"2020-08-22T03:49:46.596831+0000","last_became_peered":"2020-08-22T03:49:46.596831+0000","last_unstale":"2020-08-22T03:54:14.607004+0000","last_undegraded":"2020-08-22T03:54:14.607004+0000","last_fullsized":"2020-08-22T03:54:14.607004+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_clean_scrub_stamp":"2020-08-22T03:49:16.940536+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"19'43","reported_seq":"57","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:15.007276+0000","last_change":"2020-08-22T03:54:15.007276+0000","last_active":"2020-08-22T03:54:15.007276+0000","last_peered":"2020-08-22T03:54:15.007276+0000","last_clean":"2020-08-22T03:54:15.007276+0000","last_became_active":"2020-08-22T03:49:31.547732+0000","last_became_peered":"2020-08-22T03:49:31.547732+0000","last_unstale":"2020-08-22T03:54:15.007276+0000","last_undegraded":"2020-08-22T03:54:15.007276+0000","last_fullsized":"2020-08-22T03:54:15.007276+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":43,"ondisk_log_size":43,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":"60","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:15.007245+0000","last_change":"2020-08-22T03:49:20.323711+0000","last_active":"2020-08-22T03:54:15.007245+0000","last_peered":"2020-08-22T03:54:15.007245+0000","last_clean":"2020-08-22T03:54:15.007245+0000","last_became_active":"2020-08-22T03:49:20.323711+0000","last_became_peered":"2020-08-22T03:49:20.323711+0000","last_unstale":"2020-08-22T03:54:15.007245+0000","last_undegraded":"2020-08-22T03:54:15.007245+0000","last_fullsized":"2020-08-22T03:54:15.007245+0000","mapping_epoch":24,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":25,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_clean_scrub_stamp":"2020-08-22T03:49:16.940536+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"19'56","reported_seq":"57","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:14.867146+0000","last_change":"2020-08-22T03:49:31.550741+0000","last_active":"2020-08-22T03:54:14.867146+0000","last_peered":"2020-08-22T03:54:14.867146+0000","last_clean":"2020-08-22T03:54:14.867146+0000","last_became_active":"2020-08-22T03:49:31.550741+0000","last_became_peered":"2020-08-22T03:49:31.550741+0000","last_unstale":"2020-08-22T03:54:14.867146+0000","last_undegraded":"2020-08-22T03:54:14.867146+0000","last_fullsized":"2020-08-22T03:54:14.867146+0000","mapping_epoch":24,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":25,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":56,"ondisk_log_size":56,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"19'57","reported_seq":"57","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:15.007293+0000","last_change":"2020-08-22T03:54:15.007293+0000","last_active":"2020-08-22T03:54:15.007293+0000","last_peered":"2020-08-22T03:54:15.007293+0000","last_clean":"2020-08-22T03:54:15.007293+0000","last_became_active":"2020-08-22T03:49:31.547748+0000","last_became_peered":"2020-08-22T03:49:31.547748+0000","last_unstale":"2020-08-22T03:54:15.007293+0000","last_undegraded":"2020-08-22T03:54:15.007293+0000","last_fullsized":"2020-08-22T03:54:15.007293+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":57,"ondisk_log_size":57,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":"60","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:14.867108+0000","last_change":"2020-08-22T03:49:20.178704+0000","last_active":"2020-08-22T03:54:14.867108+0000","last_peered":"2020-08-22T03:54:14.867108+0000","last_clean":"2020-08-22T03:54:14.867108+0000","last_became_active":"2020-08-22T03:49:20.178704+0000","last_became_peered":"2020-08-22T03:49:20.178704+0000","last_unstale":"2020-08-22T03:54:14.867108+0000","last_undegraded":"2020-08-22T03:54:14.867108+0000","last_fullsized":"2020-08-22T03:54:14.867108+0000","mapping_epoch":24,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":25,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_clean_scrub_stamp":"2020-08-22T03:49:16.940536+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":"60","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:14.733870+0000","last_change":"2020-08-22T03:54:14.733870+0000","last_active":"2020-08-22T03:54:14.733870+0000","last_peered":"2020-08-22T03:54:14.733870+0000","last_clean":"2020-08-22T03:54:14.733870+0000","last_became_active":"2020-08-22T03:50:16.611490+0000","last_became_peered":"2020-08-22T03:50:16.611490+0000","last_unstale":"2020-08-22T03:54:14.733870+0000","last_undegraded":"2020-08-22T03:54:14.733870+0000","last_fullsized":"2020-08-22T03:54:14.733870+0000","mapping_epoch":34,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":35,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_clean_scrub_stamp":"2020-08-22T03:49:16.940536+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.6","version":"19'42","reported_seq":"57","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:14.867666+0000","last_change":"2020-08-22T03:54:14.867666+0000","last_active":"2020-08-22T03:54:14.867666+0000","last_peered":"2020-08-22T03:54:14.867666+0000","last_clean":"2020-08-22T03:54:14.867666+0000","last_became_active":"2020-08-22T03:49:31.550756+0000","last_became_peered":"2020-08-22T03:49:31.550756+0000","last_unstale":"2020-08-22T03:54:14.867666+0000","last_undegraded":"2020-08-22T03:54:14.867666+0000","last_fullsized":"2020-08-22T03:54:14.867666+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":42,"ondisk_log_size":42,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":826,"ondisk_log_size":826,"up":48,"acting":48,"num_store_stats":0},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1,"ondisk_log_size":1,"up":3,"acting":3,"num_store_stats":0},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":0}],"osd_stats":[{"osd":3,"up_from":24,"seq":103079215167,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":24,"seq":103079215167,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":8,"seq":34359738431,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":33,"seq":141733920831,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}]}} 2020-08-22T03:56:58.526 WARNING:scrape:Ignoring malparsed backtrace: File "src/gevent/greenlet.py", line 709, in gevent._greenlet.Greenlet.get , File "src/gevent/greenlet.py", line 317, in gevent._greenlet.Greenlet._raise_exception , File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/lib/python3.6/site-packages/gevent/_compat.py", line 47, in reraise 2020-08-22T03:56:58.526 WARNING:scrape:Ignoring malparsed backtrace: teuthology.exceptions.CommandCrashedError: Command crashed: 'CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --max-ops 400000 --objects 1024 --max-in-flight 64 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 600 --op read 100 --op write 50 --op delete 10 --op write_excl 50 --pool unique_pool_0' , 2020-08-22T03:54:44.981 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits... , 2020-08-22T03:54:44.982 INFO:teuthology.orchestra.run.smithi092:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits 2020-08-22T03:56:58.527 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:54:47.212 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-test x86_64 2:16.0.0-4564.ge64d8132f64.el8 @ceph 204 M , 2020-08-22T03:54:47.212 INFO:teuthology.orchestra.run.smithi092.stdout:Removing unused dependencies: , 2020-08-22T03:54:47.212 INFO:teuthology.orchestra.run.smithi092.stdout: jq x86_64 1.5-12.el8 @CentOS-AppStream 359 k 2020-08-22T03:56:58.527 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:54:49.701 INFO:teuthology.orchestra.run.smithi092.stdout:Freed space: 97 M , 2020-08-22T03:54:49.702 INFO:teuthology.orchestra.run.smithi092.stdout:Running transaction check , 2020-08-22T03:54:49.703 INFO:teuthology.orchestra.run.smithi092.stdout:Transaction check succeeded. 2020-08-22T03:56:58.528 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:54:52.586 INFO:teuthology.orchestra.run.smithi183.stdout: libgfortran x86_648.3.1-4.5.el8 @CentOS-Base 2.5 M , 2020-08-22T03:54:52.586 INFO:teuthology.orchestra.run.smithi183.stdout: liboath x86_642.6.2-3.el8 @epel 125 k , 2020-08-22T03:54:52.586 INFO:teuthology.orchestra.run.smithi183.stdout: libquadmath x86_648.3.1-4.5.el8 @CentOS-Base 299 k 2020-08-22T03:56:58.529 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:54:52.664 INFO:teuthology.orchestra.run.smithi092.stdout: libgfortran x86_648.3.1-4.5.el8 @CentOS-Base 2.5 M , 2020-08-22T03:54:52.664 INFO:teuthology.orchestra.run.smithi092.stdout: liboath x86_642.6.2-3.el8 @epel 125 k , 2020-08-22T03:54:52.664 INFO:teuthology.orchestra.run.smithi092.stdout: libquadmath x86_648.3.1-4.5.el8 @CentOS-Base 299 k 2020-08-22T03:56:58.530 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:54:54.568 INFO:teuthology.orchestra.run.smithi092.stdout: Erasing : ceph-mgr-dashboard-2:16.0.0-4564.ge64d8132f64.el8. 5/85 , 2020-08-22T03:54:54.585 INFO:teuthology.orchestra.run.smithi183.stdout: Erasing : python3-pecan-1.3.2-9.el8.noarch 4/85 , 2020-08-22T03:54:54.585 INFO:teuthology.orchestra.run.smithi183.stdout: Erasing : ceph-mgr-dashboard-2:16.0.0-4564.ge64d8132f64.el8. 5/85 2020-08-22T03:56:58.530 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:55:02.759 INFO:teuthology.orchestra.run.smithi183.stdout: Erasing : python3-certifi-2018.10.15-7.el8.noarch 48/85 , 2020-08-22T03:55:02.807 INFO:teuthology.orchestra.run.smithi092.stdout: Erasing : ceph-grafana-dashboards-2:16.0.0-4564.ge64d8132f64 50/85 , 2020-08-22T03:55:02.923 INFO:teuthology.orchestra.run.smithi092.stdout: Erasing : ceph-prometheus-alerts-2:16.0.0-4564.ge64d8132f64. 51/85 2020-08-22T03:56:58.531 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:55:22.603 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: libquadmath-8.3.1-4.5.el8.x86_64 77/85 , 2020-08-22T03:55:22.619 INFO:teuthology.orchestra.run.smithi183.stdout: Erasing : python3-rados-2:16.0.0-4564.ge64d8132f64.el8.x86_6 76/85 , 2020-08-22T03:55:22.627 INFO:teuthology.orchestra.run.smithi183.stdout: Running scriptlet: libquadmath-8.3.1-4.5.el8.x86_64 77/85 2020-08-22T03:56:58.532 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:55:24.187 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-urllib3-1.24.2-2.el8.noarch 77/85 , 2020-08-22T03:55:24.188 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-waitress-1.2.1-2.el8.1.noarch 78/85 , 2020-08-22T03:55:24.189 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-webencodings-0.5.1-6.el8.noarch 79/85 2020-08-22T03:56:58.532 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:55:25.495 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-grafana-dashboards-2:16.0.0-4564.ge64d8132f64.el8.noarch , 2020-08-22T03:55:25.495 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-mgr-2:16.0.0-4564.ge64d8132f64.el8.x86_64 , 2020-08-22T03:55:25.495 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-mgr-cephadm-2:16.0.0-4564.ge64d8132f64.el8.noarch 2020-08-22T03:56:58.533 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:55:26.146 INFO:teuthology.orchestra.run.smithi183.stdout: liboath-2.6.2-3.el8.x86_64 , 2020-08-22T03:55:26.146 INFO:teuthology.orchestra.run.smithi183.stdout: libquadmath-8.3.1-4.5.el8.x86_64 , 2020-08-22T03:55:26.146 INFO:teuthology.orchestra.run.smithi183.stdout: librabbitmq-0.9.0-1.el8.x86_64 2020-08-22T03:56:58.533 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:55:26.818 INFO:teuthology.orchestra.run.smithi183.stdout: , 2020-08-22T03:55:26.818 INFO:teuthology.orchestra.run.smithi183.stdout:Freed space: 185 k , 2020-08-22T03:55:26.819 INFO:teuthology.orchestra.run.smithi183.stdout:Running transaction check 2020-08-22T03:56:58.534 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:55:31.092 INFO:teuthology.orchestra.run.smithi092.stdout:Nothing to do. , 2020-08-22T03:55:31.093 INFO:teuthology.orchestra.run.smithi092.stdout:Complete! , 2020-08-22T03:55:31.156 INFO:teuthology.orchestra.run.smithi183.stdout:No match for argument: ceph-mgr-dashboard 2020-08-22T03:56:58.535 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:55:34.096 INFO:teuthology.orchestra.run.smithi183.stdout: Verifying : fuse-common-3.2.1-12.el8.x86_64 3/3 , 2020-08-22T03:55:34.096 INFO:teuthology.orchestra.run.smithi183.stdout: , 2020-08-22T03:55:34.096 INFO:teuthology.orchestra.run.smithi183.stdout:Removed: 2020-08-22T03:56:58.535 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:55:37.380 INFO:teuthology.orchestra.run.smithi092.stdout:Dependencies resolved. , 2020-08-22T03:55:37.381 INFO:teuthology.orchestra.run.smithi092.stdout:================================================================================ , 2020-08-22T03:55:37.382 INFO:teuthology.orchestra.run.smithi092.stdout: Package Arch Version Repository Size 2020-08-22T03:56:58.536 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:55:39.358 INFO:teuthology.orchestra.run.smithi183.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 4/7 , 2020-08-22T03:55:39.358 INFO:teuthology.orchestra.run.smithi183.stdout: Verifying : qemu-kvm-block-rbd-15:2.12.0-88.module_el8.1.0+297+d 5/7 , 2020-08-22T03:55:39.358 INFO:teuthology.orchestra.run.smithi183.stdout: Verifying : rbd-fuse-2:16.0.0-4564.ge64d8132f64.el8.x86_64 6/7 2020-08-22T03:56:58.537 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:55:43.855 INFO:teuthology.orchestra.run.smithi092:> sudo rm -f /etc/yum.repos.d/ceph.repo , 2020-08-22T03:55:43.921 INFO:teuthology.orchestra.run.smithi092:> sudo yum clean expire-cache , 2020-08-22T03:55:44.009 INFO:teuthology.orchestra.run.smithi183.stdout:51 files removed 2020-08-22T03:56:58.537 WARNING:scrape:Ignoring malparsed backtrace: next(self.gen) , File "/home/teuthworker/src/github.com_tchaikov_ceph_wip-crimson-recovery-qa/qa/tasks/rados.py", line 277, in task , running.get() 2020-08-22T03:56:58.568 WARNING:scrape:Ignoring malparsed backtrace: ceph version 16.0.0-4564-ge64d8132f64 (e64d8132f648167078f5841ad65b42669a68d2cb) pacific (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe0) [0x7ff89e69a39e] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x67d) [0x55dea91d2add] 2020-08-22T03:56:58.570 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:50:14.250 INFO:teuthology.orchestra.run.smithi183.stdout: , 2020-08-22T03:50:14.250 INFO:teuthology.orchestra.run.smithi183.stdout:{"epoch":32,"fsid":"9972e83e-5caf-455d-81f6-10e7c5bcb3e2","created":"2020-08-22T03:49:05.967889+0000","modified":"2020-08-22T03:50:11.943469+0000","last_up_change":"2020-08-22T03:49:42.013320+0000","last_in_change":"2020-08-22T03:49:11.484438+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-22T03:49:09.511038+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-22T03:49:16.052382+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-22T03:49:28.065828+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"e0d320b8-4993-4b1a-b741-ab771add082d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6800","nonce":25504}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6801","nonce":25504}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6806","nonce":25504}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6807","nonce":25504}]},"public_addr":"172.21.15.183:6800/25504","cluster_addr":"172.21.15.183:6801/25504","heartbeat_back_addr":"172.21.15.183:6806/25504","heartbeat_front_addr":"172.21.15.183:6807/25504","state":["exists","up"]},{"osd":1,"uuid":"51144378-a6ca-485c-8a15-20c001a1ed60","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":18,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6802","nonce":25547}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6803","nonce":25547}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6804","nonce":25547}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6805","nonce":25547}]},"public_addr":"172.21.15.183:6802/25547","cluster_addr":"172.21.15.183:6803/25547","heartbeat_back_addr":"172.21.15.183:6804/25547","heartbeat_front_addr":"172.21.15.183:6805/25547","state":["exists"]},{"osd":2,"uuid":"9f4f44e0-4c21-49c7-83c6-9fd9c902a564","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":23,"up_from":24,"up_thru":24,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6802","nonce":25353}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6803","nonce":25353}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6806","nonce":25353}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6807","nonce":25353}]},"public_addr":"172.21.15.92:6802/25353","cluster_addr":"172.21.15.92:6803/25353","heartbeat_back_addr":"172.21.15.92:6806/25353","heartbeat_front_addr":"172.21.15.92:6807/25353","state":["exists","up"]},{"osd":3,"uuid":"39fb66cd-8671-4a52-a6bd-6376d5ff4f5b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":23,"up_from":24,"up_thru":31,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6804","nonce":25397}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6805","nonce":25397}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6808","nonce":25397}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6809","nonce":25397}]},"public_addr":"172.21.15.92:6804/25397","cluster_addr":"172.21.15.92:6805/25397","heartbeat_back_addr":"172.21.15.92:6808/25397","heartbeat_front_addr":"172.21.15.92:6809/25397","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-22T03:49:41.010632+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"2020-08-22T03:49:41.010632+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-22T03:49:41.010632+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.2","mappings":[{"from":0,"to":3}]},{"pgid":"2.4","mappings":[{"from":2,"to":1}]},{"pgid":"2.7","mappings":[{"from":2,"to":3}]}],"pg_temp":[{"pgid":"2.2","osds":[3,2,0]},{"pgid":"2.4","osds":[3,0,2]},{"pgid":"2.7","osds":[0,3,2]}],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-22T03:50:15.942+0000 7f343b7fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.0 v2:172.21.15.183:6800/25504 for 24sec e20 v20) 2020-08-22T03:56:58.572 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:51:43.036 INFO:teuthology.orchestra.run.smithi183.stdout:{"epoch":32,"fsid":"9972e83e-5caf-455d-81f6-10e7c5bcb3e2","created":"2020-08-22T03:49:05.967889+0000","modified":"2020-08-22T03:50:11.943469+0000","last_up_change":"2020-08-22T03:49:42.013320+0000","last_in_change":"2020-08-22T03:49:11.484438+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-22T03:49:09.511038+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-22T03:49:16.052382+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-22T03:49:28.065828+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"e0d320b8-4993-4b1a-b741-ab771add082d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6800","nonce":25504}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6801","nonce":25504}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6806","nonce":25504}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6807","nonce":25504}]},"public_addr":"172.21.15.183:6800/25504","cluster_addr":"172.21.15.183:6801/25504","heartbeat_back_addr":"172.21.15.183:6806/25504","heartbeat_front_addr":"172.21.15.183:6807/25504","state":["exists","up"]},{"osd":1,"uuid":"51144378-a6ca-485c-8a15-20c001a1ed60","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":18,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6802","nonce":25547}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6803","nonce":25547}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6804","nonce":25547}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6805","nonce":25547}]},"public_addr":"172.21.15.183:6802/25547","cluster_addr":"172.21.15.183:6803/25547","heartbeat_back_addr":"172.21.15.183:6804/25547","heartbeat_front_addr":"172.21.15.183:6805/25547","state":["exists"]},{"osd":2,"uuid":"9f4f44e0-4c21-49c7-83c6-9fd9c902a564","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":23,"up_from":24,"up_thru":24,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6802","nonce":25353}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6803","nonce":25353}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6806","nonce":25353}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6807","nonce":25353}]},"public_addr":"172.21.15.92:6802/25353","cluster_addr":"172.21.15.92:6803/25353","heartbeat_back_addr":"172.21.15.92:6806/25353","heartbeat_front_addr":"172.21.15.92:6807/25353","state":["exists","up"]},{"osd":3,"uuid":"39fb66cd-8671-4a52-a6bd-6376d5ff4f5b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":23,"up_from":24,"up_thru":31,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6804","nonce":25397}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6805","nonce":25397}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6808","nonce":25397}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6809","nonce":25397}]},"public_addr":"172.21.15.92:6804/25397","cluster_addr":"172.21.15.92:6805/25397","heartbeat_back_addr":"172.21.15.92:6808/25397","heartbeat_front_addr":"172.21.15.92:6809/25397","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-22T03:49:41.010632+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"2020-08-22T03:49:41.010632+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-22T03:49:41.010632+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.2","mappings":[{"from":0,"to":3}]},{"pgid":"2.4","mappings":[{"from":2,"to":1}]},{"pgid":"2.7","mappings":[{"from":2,"to":3}]}],"pg_temp":[{"pgid":"2.2","osds":[3,2,0]},{"pgid":"2.4","osds":[3,0,2]},{"pgid":"2.7","osds":[0,3,2]}],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-22T03:51:45.951+0000 7f343b7fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.0 v2:172.21.15.183:6800/25504 for 24sec e20 v20) , 2020-08-22T03:51:46.046 INFO:teuthology.orchestra.run.smithi183:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json 2020-08-22T03:56:58.574 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:53:11.768 INFO:teuthology.orchestra.run.smithi183.stdout:{"epoch":32,"fsid":"9972e83e-5caf-455d-81f6-10e7c5bcb3e2","created":"2020-08-22T03:49:05.967889+0000","modified":"2020-08-22T03:50:11.943469+0000","last_up_change":"2020-08-22T03:49:42.013320+0000","last_in_change":"2020-08-22T03:49:11.484438+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-22T03:49:09.511038+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-22T03:49:16.052382+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-22T03:49:28.065828+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"e0d320b8-4993-4b1a-b741-ab771add082d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6800","nonce":25504}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6801","nonce":25504}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6806","nonce":25504}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6807","nonce":25504}]},"public_addr":"172.21.15.183:6800/25504","cluster_addr":"172.21.15.183:6801/25504","heartbeat_back_addr":"172.21.15.183:6806/25504","heartbeat_front_addr":"172.21.15.183:6807/25504","state":["exists","up"]},{"osd":1,"uuid":"51144378-a6ca-485c-8a15-20c001a1ed60","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":18,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6802","nonce":25547}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6803","nonce":25547}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6804","nonce":25547}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.183:6805","nonce":25547}]},"public_addr":"172.21.15.183:6802/25547","cluster_addr":"172.21.15.183:6803/25547","heartbeat_back_addr":"172.21.15.183:6804/25547","heartbeat_front_addr":"172.21.15.183:6805/25547","state":["exists"]},{"osd":2,"uuid":"9f4f44e0-4c21-49c7-83c6-9fd9c902a564","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":23,"up_from":24,"up_thru":24,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6802","nonce":25353}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6803","nonce":25353}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6806","nonce":25353}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6807","nonce":25353}]},"public_addr":"172.21.15.92:6802/25353","cluster_addr":"172.21.15.92:6803/25353","heartbeat_back_addr":"172.21.15.92:6806/25353","heartbeat_front_addr":"172.21.15.92:6807/25353","state":["exists","up"]},{"osd":3,"uuid":"39fb66cd-8671-4a52-a6bd-6376d5ff4f5b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":23,"up_from":24,"up_thru":31,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6804","nonce":25397}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6805","nonce":25397}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6808","nonce":25397}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.92:6809","nonce":25397}]},"public_addr":"172.21.15.92:6804/25397","cluster_addr":"172.21.15.92:6805/25397","heartbeat_back_addr":"172.21.15.92:6808/25397","heartbeat_front_addr":"172.21.15.92:6809/25397","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-22T03:49:41.010632+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"2020-08-22T03:49:41.010632+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-22T03:49:41.010632+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.2","mappings":[{"from":0,"to":3}]},{"pgid":"2.4","mappings":[{"from":2,"to":1}]},{"pgid":"2.7","mappings":[{"from":2,"to":3}]}],"pg_temp":[{"pgid":"2.2","osds":[3,2,0]},{"pgid":"2.4","osds":[3,0,2]},{"pgid":"2.7","osds":[0,3,2]}],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-22T03:53:14.780 INFO:teuthology.orchestra.run.smithi183:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-22T03:53:15.052 INFO:teuthology.orchestra.run.smithi183.stdout: 2020-08-22T03:56:58.576 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:54:17.718 INFO:teuthology.orchestra.run.smithi183.stdout: , dumped all , 2020-08-22T03:54:17.732 INFO:teuthology.orchestra.run.smithi183.stdout:{"pg_ready":true,"pg_map":{"version":183,"stamp":"2020-08-22T03:54:17.521444+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":827,"ondisk_log_size":827,"up":75,"acting":75,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":0,"num_osds":4,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"3.f","version":"19'50","reported_seq":"58","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:14.607086+0000","last_change":"2020-08-22T03:54:14.607086+0000","last_active":"2020-08-22T03:54:14.607086+0000","last_peered":"2020-08-22T03:54:14.607086+0000","last_clean":"2020-08-22T03:54:14.607086+0000","last_became_active":"2020-08-22T03:49:46.597567+0000","last_became_peered":"2020-08-22T03:49:46.597567+0000","last_unstale":"2020-08-22T03:54:14.607086+0000","last_undegraded":"2020-08-22T03:54:14.607086+0000","last_fullsized":"2020-08-22T03:54:14.607086+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":50,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"19'46","reported_seq":"57","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:14.867685+0000","last_change":"2020-08-22T03:54:14.867685+0000","last_active":"2020-08-22T03:54:14.867685+0000","last_peered":"2020-08-22T03:54:14.867685+0000","last_clean":"2020-08-22T03:54:14.867685+0000","last_became_active":"2020-08-22T03:49:31.550771+0000","last_became_peered":"2020-08-22T03:49:31.550771+0000","last_unstale":"2020-08-22T03:54:14.867685+0000","last_undegraded":"2020-08-22T03:54:14.867685+0000","last_fullsized":"2020-08-22T03:54:14.867685+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":46,"ondisk_log_size":46,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"19'47","reported_seq":"57","reported_epoch":"35","state":"active+clean+wait","last_fresh":"2020-08-22T03:54:14.733949+0000","last_change":"2020-08-22T03:54:14.733949+0000","last_active":"2020-08-22T03:54:14.733949+0000","last_peered":"2020-08-22T03:54:14.733949+0000","last_clean":"2020-08-22T03:54:14.733949+0000","last_became_active":"2020-08-22T03:54:14.733949+0000","last_became_peered":"2020-08-22T03:54:14.733949+0000","last_unstale":"2020-08-22T03:54:14.733949+0000","last_undegraded":"2020-08-22T03:54:14.733949+0000","last_fullsized":"2020-08-22T03:54:14.733949+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":47,"ondisk_log_size":47,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"19'46","reported_seq":"57","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:14.607068+0000","last_change":"2020-08-22T03:49:46.597549+0000","last_active":"2020-08-22T03:54:14.607068+0000","last_peered":"2020-08-22T03:54:14.607068+0000","last_clean":"2020-08-22T03:54:14.607068+0000","last_became_active":"2020-08-22T03:49:46.597549+0000","last_became_peered":"2020-08-22T03:49:46.597549+0000","last_unstale":"2020-08-22T03:54:14.607068+0000","last_undegraded":"2020-08-22T03:54:14.607068+0000","last_fullsized":"2020-08-22T03:54:14.607068+0000","mapping_epoch":24,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":27,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":46,"ondisk_log_size":46,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"18'33","reported_seq":"57","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:15.007309+0000","last_change":"2020-08-22T03:49:31.547763+0000","last_active":"2020-08-22T03:54:15.007309+0000","last_peered":"2020-08-22T03:54:15.007309+0000","last_clean":"2020-08-22T03:54:15.007309+0000","last_became_active":"2020-08-22T03:49:31.547763+0000","last_became_peered":"2020-08-22T03:49:31.547763+0000","last_unstale":"2020-08-22T03:54:15.007309+0000","last_undegraded":"2020-08-22T03:54:15.007309+0000","last_fullsized":"2020-08-22T03:54:15.007309+0000","mapping_epoch":24,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":25,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":33,"ondisk_log_size":33,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"19'59","reported_seq":"57","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:14.607051+0000","last_change":"2020-08-22T03:54:14.607051+0000","last_active":"2020-08-22T03:54:14.607051+0000","last_peered":"2020-08-22T03:54:14.607051+0000","last_clean":"2020-08-22T03:54:14.607051+0000","last_became_active":"2020-08-22T03:49:46.597450+0000","last_became_peered":"2020-08-22T03:49:46.597450+0000","last_unstale":"2020-08-22T03:54:14.607051+0000","last_undegraded":"2020-08-22T03:54:14.607051+0000","last_fullsized":"2020-08-22T03:54:14.607051+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":59,"ondisk_log_size":59,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"19'81","reported_seq":"57","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:14.607032+0000","last_change":"2020-08-22T03:54:14.607032+0000","last_active":"2020-08-22T03:54:14.607032+0000","last_peered":"2020-08-22T03:54:14.607032+0000","last_clean":"2020-08-22T03:54:14.607032+0000","last_became_active":"2020-08-22T03:49:46.597349+0000","last_became_peered":"2020-08-22T03:49:46.597349+0000","last_unstale":"2020-08-22T03:54:14.607032+0000","last_undegraded":"2020-08-22T03:54:14.607032+0000","last_fullsized":"2020-08-22T03:54:14.607032+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":81,"ondisk_log_size":81,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"19'56","reported_seq":"57","reported_epoch":"35","state":"active+clean+wait","last_fresh":"2020-08-22T03:54:14.733930+0000","last_change":"2020-08-22T03:54:14.733930+0000","last_active":"2020-08-22T03:54:14.733930+0000","last_peered":"2020-08-22T03:54:14.733930+0000","last_clean":"2020-08-22T03:54:14.733930+0000","last_became_active":"2020-08-22T03:54:14.733930+0000","last_became_peered":"2020-08-22T03:54:14.733930+0000","last_unstale":"2020-08-22T03:54:14.733930+0000","last_undegraded":"2020-08-22T03:54:14.733930+0000","last_fullsized":"2020-08-22T03:54:14.733930+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":56,"ondisk_log_size":56,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":"7","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:14.733853+0000","last_change":"2020-08-22T03:54:14.733853+0000","last_active":"2020-08-22T03:54:14.733853+0000","last_peered":"2020-08-22T03:54:14.733853+0000","last_clean":"2020-08-22T03:54:14.733853+0000","last_became_active":"2020-08-22T03:54:14.733853+0000","last_became_peered":"2020-08-22T03:54:14.733853+0000","last_unstale":"2020-08-22T03:54:14.733853+0000","last_undegraded":"2020-08-22T03:54:14.733853+0000","last_fullsized":"2020-08-22T03:54:14.733853+0000","mapping_epoch":34,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":35,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_clean_scrub_stamp":"2020-08-22T03:49:16.940536+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"19'41","reported_seq":"57","reported_epoch":"35","state":"active+clean+wait","last_fresh":"2020-08-22T03:54:14.733905+0000","last_change":"2020-08-22T03:54:14.733905+0000","last_active":"2020-08-22T03:54:14.733905+0000","last_peered":"2020-08-22T03:54:14.733905+0000","last_clean":"2020-08-22T03:54:14.733905+0000","last_became_active":"2020-08-22T03:54:14.733905+0000","last_became_peered":"2020-08-22T03:54:14.733905+0000","last_unstale":"2020-08-22T03:54:14.733905+0000","last_undegraded":"2020-08-22T03:54:14.733905+0000","last_fullsized":"2020-08-22T03:54:14.733905+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":41,"ondisk_log_size":41,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":"60","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:15.007212+0000","last_change":"2020-08-22T03:54:15.007212+0000","last_active":"2020-08-22T03:54:15.007212+0000","last_peered":"2020-08-22T03:54:15.007212+0000","last_clean":"2020-08-22T03:54:15.007212+0000","last_became_active":"2020-08-22T03:50:16.579309+0000","last_became_peered":"2020-08-22T03:50:16.579309+0000","last_unstale":"2020-08-22T03:54:15.007212+0000","last_undegraded":"2020-08-22T03:54:15.007212+0000","last_fullsized":"2020-08-22T03:54:15.007212+0000","mapping_epoch":34,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":35,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_clean_scrub_stamp":"2020-08-22T03:49:16.940536+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.3","version":"19'53","reported_seq":"57","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:14.867129+0000","last_change":"2020-08-22T03:49:31.550725+0000","last_active":"2020-08-22T03:54:14.867129+0000","last_peered":"2020-08-22T03:54:14.867129+0000","last_clean":"2020-08-22T03:54:14.867129+0000","last_became_active":"2020-08-22T03:49:31.550725+0000","last_became_peered":"2020-08-22T03:49:31.550725+0000","last_unstale":"2020-08-22T03:54:14.867129+0000","last_undegraded":"2020-08-22T03:54:14.867129+0000","last_fullsized":"2020-08-22T03:54:14.867129+0000","mapping_epoch":24,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":25,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":53,"ondisk_log_size":53,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":"60","reported_epoch":"35","state":"active+clean+wait","last_fresh":"2020-08-22T03:54:14.733805+0000","last_change":"2020-08-22T03:54:14.733805+0000","last_active":"2020-08-22T03:54:14.733805+0000","last_peered":"2020-08-22T03:54:14.733805+0000","last_clean":"2020-08-22T03:54:14.733805+0000","last_became_active":"2020-08-22T03:54:14.733805+0000","last_became_peered":"2020-08-22T03:54:14.733805+0000","last_unstale":"2020-08-22T03:54:14.733805+0000","last_undegraded":"2020-08-22T03:54:14.733805+0000","last_fullsized":"2020-08-22T03:54:14.733805+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_clean_scrub_stamp":"2020-08-22T03:49:16.940536+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"19'66","reported_seq":"57","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:15.007260+0000","last_change":"2020-08-22T03:54:15.007260+0000","last_active":"2020-08-22T03:54:15.007260+0000","last_peered":"2020-08-22T03:54:15.007260+0000","last_clean":"2020-08-22T03:54:15.007260+0000","last_became_active":"2020-08-22T03:49:31.547716+0000","last_became_peered":"2020-08-22T03:49:31.547716+0000","last_unstale":"2020-08-22T03:54:15.007260+0000","last_undegraded":"2020-08-22T03:54:15.007260+0000","last_fullsized":"2020-08-22T03:54:15.007260+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":66,"ondisk_log_size":66,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":"60","reported_epoch":"35","state":"active+clean+wait","last_fresh":"2020-08-22T03:54:14.733835+0000","last_change":"2020-08-22T03:54:14.733835+0000","last_active":"2020-08-22T03:54:14.733835+0000","last_peered":"2020-08-22T03:54:14.733835+0000","last_clean":"2020-08-22T03:54:14.733835+0000","last_became_active":"2020-08-22T03:49:46.597044+0000","last_became_peered":"2020-08-22T03:49:46.597044+0000","last_unstale":"2020-08-22T03:54:14.733835+0000","last_undegraded":"2020-08-22T03:54:14.733835+0000","last_fullsized":"2020-08-22T03:54:14.733835+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_clean_scrub_stamp":"2020-08-22T03:49:16.940536+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"9'1","reported_seq":"61","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:15.007189+0000","last_change":"2020-08-22T03:49:16.536195+0000","last_active":"2020-08-22T03:54:15.007189+0000","last_peered":"2020-08-22T03:54:15.007189+0000","last_clean":"2020-08-22T03:54:15.007189+0000","last_became_active":"2020-08-22T03:49:16.536195+0000","last_became_peered":"2020-08-22T03:49:16.536195+0000","last_unstale":"2020-08-22T03:54:15.007189+0000","last_undegraded":"2020-08-22T03:54:15.007189+0000","last_fullsized":"2020-08-22T03:54:15.007189+0000","mapping_epoch":24,"log_start":"0'0","ondisk_log_start":"0'0","created":7,"last_epoch_clean":25,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:10.478504+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:10.478504+0000","last_clean_scrub_stamp":"2020-08-22T03:49:10.478504+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2","version":"19'50","reported_seq":"57","reported_epoch":"35","state":"active+clean+wait","last_fresh":"2020-08-22T03:54:14.733886+0000","last_change":"2020-08-22T03:54:14.733886+0000","last_active":"2020-08-22T03:54:14.733886+0000","last_peered":"2020-08-22T03:54:14.733886+0000","last_clean":"2020-08-22T03:54:14.733886+0000","last_became_active":"2020-08-22T03:49:46.597239+0000","last_became_peered":"2020-08-22T03:49:46.597239+0000","last_unstale":"2020-08-22T03:54:14.733886+0000","last_undegraded":"2020-08-22T03:54:14.733886+0000","last_fullsized":"2020-08-22T03:54:14.733886+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":50,"ondisk_log_size":50,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":"61","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:14.607004+0000","last_change":"2020-08-22T03:54:14.607004+0000","last_active":"2020-08-22T03:54:14.607004+0000","last_peered":"2020-08-22T03:54:14.607004+0000","last_clean":"2020-08-22T03:54:14.607004+0000","last_became_active":"2020-08-22T03:49:46.596831+0000","last_became_peered":"2020-08-22T03:49:46.596831+0000","last_unstale":"2020-08-22T03:54:14.607004+0000","last_undegraded":"2020-08-22T03:54:14.607004+0000","last_fullsized":"2020-08-22T03:54:14.607004+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_clean_scrub_stamp":"2020-08-22T03:49:16.940536+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"19'43","reported_seq":"57","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:15.007276+0000","last_change":"2020-08-22T03:54:15.007276+0000","last_active":"2020-08-22T03:54:15.007276+0000","last_peered":"2020-08-22T03:54:15.007276+0000","last_clean":"2020-08-22T03:54:15.007276+0000","last_became_active":"2020-08-22T03:49:31.547732+0000","last_became_peered":"2020-08-22T03:49:31.547732+0000","last_unstale":"2020-08-22T03:54:15.007276+0000","last_undegraded":"2020-08-22T03:54:15.007276+0000","last_fullsized":"2020-08-22T03:54:15.007276+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":43,"ondisk_log_size":43,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":"60","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:15.007245+0000","last_change":"2020-08-22T03:49:20.323711+0000","last_active":"2020-08-22T03:54:15.007245+0000","last_peered":"2020-08-22T03:54:15.007245+0000","last_clean":"2020-08-22T03:54:15.007245+0000","last_became_active":"2020-08-22T03:49:20.323711+0000","last_became_peered":"2020-08-22T03:49:20.323711+0000","last_unstale":"2020-08-22T03:54:15.007245+0000","last_undegraded":"2020-08-22T03:54:15.007245+0000","last_fullsized":"2020-08-22T03:54:15.007245+0000","mapping_epoch":24,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":25,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_clean_scrub_stamp":"2020-08-22T03:49:16.940536+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.4","version":"19'56","reported_seq":"57","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:14.867146+0000","last_change":"2020-08-22T03:49:31.550741+0000","last_active":"2020-08-22T03:54:14.867146+0000","last_peered":"2020-08-22T03:54:14.867146+0000","last_clean":"2020-08-22T03:54:14.867146+0000","last_became_active":"2020-08-22T03:49:31.550741+0000","last_became_peered":"2020-08-22T03:49:31.550741+0000","last_unstale":"2020-08-22T03:54:14.867146+0000","last_undegraded":"2020-08-22T03:54:14.867146+0000","last_fullsized":"2020-08-22T03:54:14.867146+0000","mapping_epoch":24,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":25,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":56,"ondisk_log_size":56,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"19'57","reported_seq":"57","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:15.007293+0000","last_change":"2020-08-22T03:54:15.007293+0000","last_active":"2020-08-22T03:54:15.007293+0000","last_peered":"2020-08-22T03:54:15.007293+0000","last_clean":"2020-08-22T03:54:15.007293+0000","last_became_active":"2020-08-22T03:49:31.547748+0000","last_became_peered":"2020-08-22T03:49:31.547748+0000","last_unstale":"2020-08-22T03:54:15.007293+0000","last_undegraded":"2020-08-22T03:54:15.007293+0000","last_fullsized":"2020-08-22T03:54:15.007293+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":57,"ondisk_log_size":57,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":"60","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:14.867108+0000","last_change":"2020-08-22T03:49:20.178704+0000","last_active":"2020-08-22T03:54:14.867108+0000","last_peered":"2020-08-22T03:54:14.867108+0000","last_clean":"2020-08-22T03:54:14.867108+0000","last_became_active":"2020-08-22T03:49:20.178704+0000","last_became_peered":"2020-08-22T03:49:20.178704+0000","last_unstale":"2020-08-22T03:54:14.867108+0000","last_undegraded":"2020-08-22T03:54:14.867108+0000","last_fullsized":"2020-08-22T03:54:14.867108+0000","mapping_epoch":24,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":25,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_clean_scrub_stamp":"2020-08-22T03:49:16.940536+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":"60","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:14.733870+0000","last_change":"2020-08-22T03:54:14.733870+0000","last_active":"2020-08-22T03:54:14.733870+0000","last_peered":"2020-08-22T03:54:14.733870+0000","last_clean":"2020-08-22T03:54:14.733870+0000","last_became_active":"2020-08-22T03:50:16.611490+0000","last_became_peered":"2020-08-22T03:50:16.611490+0000","last_unstale":"2020-08-22T03:54:14.733870+0000","last_undegraded":"2020-08-22T03:54:14.733870+0000","last_fullsized":"2020-08-22T03:54:14.733870+0000","mapping_epoch":34,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":35,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:16.940536+0000","last_clean_scrub_stamp":"2020-08-22T03:49:16.940536+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.6","version":"19'42","reported_seq":"57","reported_epoch":"35","state":"active+clean","last_fresh":"2020-08-22T03:54:14.867666+0000","last_change":"2020-08-22T03:54:14.867666+0000","last_active":"2020-08-22T03:54:14.867666+0000","last_peered":"2020-08-22T03:54:14.867666+0000","last_clean":"2020-08-22T03:54:14.867666+0000","last_became_active":"2020-08-22T03:49:31.550756+0000","last_became_peered":"2020-08-22T03:49:31.550756+0000","last_unstale":"2020-08-22T03:54:14.867666+0000","last_undegraded":"2020-08-22T03:54:14.867666+0000","last_fullsized":"2020-08-22T03:54:14.867666+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2020-08-22T03:49:28.971761+0000","last_clean_scrub_stamp":"2020-08-22T03:49:28.971761+0000","log_size":42,"ondisk_log_size":42,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":826,"ondisk_log_size":826,"up":48,"acting":48,"num_store_stats":0},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1,"ondisk_log_size":1,"up":3,"acting":3,"num_store_stats":0},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":0}],"osd_stats":[{"osd":3,"up_from":24,"seq":103079215167,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":24,"seq":103079215167,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":8,"seq":34359738431,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":33,"seq":141733920831,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}]}} 2020-08-22T03:56:58.578 WARNING:scrape:Ignoring malparsed backtrace: File "src/gevent/greenlet.py", line 709, in gevent._greenlet.Greenlet.get , File "src/gevent/greenlet.py", line 317, in gevent._greenlet.Greenlet._raise_exception , File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/lib/python3.6/site-packages/gevent/_compat.py", line 47, in reraise 2020-08-22T03:56:58.579 WARNING:scrape:Ignoring malparsed backtrace: teuthology.exceptions.CommandCrashedError: Command crashed: 'CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --max-ops 400000 --objects 1024 --max-in-flight 64 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 600 --op read 100 --op write 50 --op delete 10 --op write_excl 50 --pool unique_pool_0' , 2020-08-22T03:54:44.981 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits... , 2020-08-22T03:54:44.982 INFO:teuthology.orchestra.run.smithi092:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits 2020-08-22T03:56:58.579 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:54:47.212 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-test x86_64 2:16.0.0-4564.ge64d8132f64.el8 @ceph 204 M , 2020-08-22T03:54:47.212 INFO:teuthology.orchestra.run.smithi092.stdout:Removing unused dependencies: , 2020-08-22T03:54:47.212 INFO:teuthology.orchestra.run.smithi092.stdout: jq x86_64 1.5-12.el8 @CentOS-AppStream 359 k 2020-08-22T03:56:58.580 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:54:49.701 INFO:teuthology.orchestra.run.smithi092.stdout:Freed space: 97 M , 2020-08-22T03:54:49.702 INFO:teuthology.orchestra.run.smithi092.stdout:Running transaction check , 2020-08-22T03:54:49.703 INFO:teuthology.orchestra.run.smithi092.stdout:Transaction check succeeded. 2020-08-22T03:56:58.581 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:54:52.586 INFO:teuthology.orchestra.run.smithi183.stdout: libgfortran x86_648.3.1-4.5.el8 @CentOS-Base 2.5 M , 2020-08-22T03:54:52.586 INFO:teuthology.orchestra.run.smithi183.stdout: liboath x86_642.6.2-3.el8 @epel 125 k , 2020-08-22T03:54:52.586 INFO:teuthology.orchestra.run.smithi183.stdout: libquadmath x86_648.3.1-4.5.el8 @CentOS-Base 299 k 2020-08-22T03:56:58.581 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:54:52.664 INFO:teuthology.orchestra.run.smithi092.stdout: libgfortran x86_648.3.1-4.5.el8 @CentOS-Base 2.5 M , 2020-08-22T03:54:52.664 INFO:teuthology.orchestra.run.smithi092.stdout: liboath x86_642.6.2-3.el8 @epel 125 k , 2020-08-22T03:54:52.664 INFO:teuthology.orchestra.run.smithi092.stdout: libquadmath x86_648.3.1-4.5.el8 @CentOS-Base 299 k 2020-08-22T03:56:58.582 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:54:54.568 INFO:teuthology.orchestra.run.smithi092.stdout: Erasing : ceph-mgr-dashboard-2:16.0.0-4564.ge64d8132f64.el8. 5/85 , 2020-08-22T03:54:54.585 INFO:teuthology.orchestra.run.smithi183.stdout: Erasing : python3-pecan-1.3.2-9.el8.noarch 4/85 , 2020-08-22T03:54:54.585 INFO:teuthology.orchestra.run.smithi183.stdout: Erasing : ceph-mgr-dashboard-2:16.0.0-4564.ge64d8132f64.el8. 5/85 2020-08-22T03:56:58.583 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:55:02.759 INFO:teuthology.orchestra.run.smithi183.stdout: Erasing : python3-certifi-2018.10.15-7.el8.noarch 48/85 , 2020-08-22T03:55:02.807 INFO:teuthology.orchestra.run.smithi092.stdout: Erasing : ceph-grafana-dashboards-2:16.0.0-4564.ge64d8132f64 50/85 , 2020-08-22T03:55:02.923 INFO:teuthology.orchestra.run.smithi092.stdout: Erasing : ceph-prometheus-alerts-2:16.0.0-4564.ge64d8132f64. 51/85 2020-08-22T03:56:58.584 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:55:22.603 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: libquadmath-8.3.1-4.5.el8.x86_64 77/85 , 2020-08-22T03:55:22.619 INFO:teuthology.orchestra.run.smithi183.stdout: Erasing : python3-rados-2:16.0.0-4564.ge64d8132f64.el8.x86_6 76/85 , 2020-08-22T03:55:22.627 INFO:teuthology.orchestra.run.smithi183.stdout: Running scriptlet: libquadmath-8.3.1-4.5.el8.x86_64 77/85 2020-08-22T03:56:58.584 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:55:24.187 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-urllib3-1.24.2-2.el8.noarch 77/85 , 2020-08-22T03:55:24.188 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-waitress-1.2.1-2.el8.1.noarch 78/85 , 2020-08-22T03:55:24.189 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-webencodings-0.5.1-6.el8.noarch 79/85 2020-08-22T03:56:58.585 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:55:25.495 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-grafana-dashboards-2:16.0.0-4564.ge64d8132f64.el8.noarch , 2020-08-22T03:55:25.495 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-mgr-2:16.0.0-4564.ge64d8132f64.el8.x86_64 , 2020-08-22T03:55:25.495 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-mgr-cephadm-2:16.0.0-4564.ge64d8132f64.el8.noarch 2020-08-22T03:56:58.586 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:55:26.146 INFO:teuthology.orchestra.run.smithi183.stdout: liboath-2.6.2-3.el8.x86_64 , 2020-08-22T03:55:26.146 INFO:teuthology.orchestra.run.smithi183.stdout: libquadmath-8.3.1-4.5.el8.x86_64 , 2020-08-22T03:55:26.146 INFO:teuthology.orchestra.run.smithi183.stdout: librabbitmq-0.9.0-1.el8.x86_64 2020-08-22T03:56:58.586 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:55:26.818 INFO:teuthology.orchestra.run.smithi183.stdout: , 2020-08-22T03:55:26.818 INFO:teuthology.orchestra.run.smithi183.stdout:Freed space: 185 k , 2020-08-22T03:55:26.819 INFO:teuthology.orchestra.run.smithi183.stdout:Running transaction check 2020-08-22T03:56:58.587 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:55:31.092 INFO:teuthology.orchestra.run.smithi092.stdout:Nothing to do. , 2020-08-22T03:55:31.093 INFO:teuthology.orchestra.run.smithi092.stdout:Complete! , 2020-08-22T03:55:31.156 INFO:teuthology.orchestra.run.smithi183.stdout:No match for argument: ceph-mgr-dashboard 2020-08-22T03:56:58.587 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:55:34.096 INFO:teuthology.orchestra.run.smithi183.stdout: Verifying : fuse-common-3.2.1-12.el8.x86_64 3/3 , 2020-08-22T03:55:34.096 INFO:teuthology.orchestra.run.smithi183.stdout: , 2020-08-22T03:55:34.096 INFO:teuthology.orchestra.run.smithi183.stdout:Removed: 2020-08-22T03:56:58.588 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:55:37.380 INFO:teuthology.orchestra.run.smithi092.stdout:Dependencies resolved. , 2020-08-22T03:55:37.381 INFO:teuthology.orchestra.run.smithi092.stdout:================================================================================ , 2020-08-22T03:55:37.382 INFO:teuthology.orchestra.run.smithi092.stdout: Package Arch Version Repository Size 2020-08-22T03:56:58.589 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:55:39.358 INFO:teuthology.orchestra.run.smithi183.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 4/7 , 2020-08-22T03:55:39.358 INFO:teuthology.orchestra.run.smithi183.stdout: Verifying : qemu-kvm-block-rbd-15:2.12.0-88.module_el8.1.0+297+d 5/7 , 2020-08-22T03:55:39.358 INFO:teuthology.orchestra.run.smithi183.stdout: Verifying : rbd-fuse-2:16.0.0-4564.ge64d8132f64.el8.x86_64 6/7 2020-08-22T03:56:58.589 WARNING:scrape:Ignoring malparsed backtrace: 2020-08-22T03:55:43.855 INFO:teuthology.orchestra.run.smithi092:> sudo rm -f /etc/yum.repos.d/ceph.repo , 2020-08-22T03:55:43.921 INFO:teuthology.orchestra.run.smithi092:> sudo yum clean expire-cache , 2020-08-22T03:55:44.009 INFO:teuthology.orchestra.run.smithi183.stdout:51 files removed 2020-08-22T03:56:58.590 WARNING:scrape:Ignoring malparsed backtrace: next(self.gen) , File "/home/teuthworker/src/github.com_tchaikov_ceph_wip-crimson-recovery-qa/qa/tasks/rados.py", line 277, in task , running.get() 2020-08-22T03:56:58.611 INFO:scrape:Found 1 distinct failure reasons 2020-08-22T03:56:58.613 INFO:scrape:Failure: Command crashed: 'CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --max-ops 400000 --objects 1024 --max-in-flight 64 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 600 --op read 100 --op write 50 --op delete 10 --op write_excl 50 --pool unique_pool_0' 2020-08-22T03:56:58.613 INFO:scrape:1 jobs: ['5365998'] 2020-08-22T03:56:58.614 INFO:scrape:suites: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{default}', 'centos_latest', 'clusters/{fixed-2}', 'crimson-rados:thrash/{0-size-min-size-overrides/3-size-2-min-size', 'deploy/ceph', 'objectstore/bluestore', 'thrashers/default', 'thrashosds-health', 'workloads/small-objects}'] 2020-08-22T03:56:58.614 INFO:scrape: 2020-08-22T03:56:58.615 INFO:teuthology.results:Sending results to tchaikov@gmail.com: Test Run: kchai-2020-08-22_03:33:19-crimson-rados:thrash-wip-crimson-more-ops-kefu-distro-basic-smithi ================================================================= info: http://pulpito.ceph.com/kchai-2020-08-22_03:33:19-crimson-rados:thrash-wip-crimson-more-ops-kefu-distro-basic-smithi/ logs: http://qa-proxy.ceph.com/teuthology/kchai-2020-08-22_03:33:19-crimson-rados:thrash-wip-crimson-more-ops-kefu-distro-basic-smithi/ failed: 1 dead: 0 running: 0 waiting: 0 queued: 0 passed: 0 Fail ================================================================= [5365998] crimson-rados:thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{default} centos_latest clusters/{fixed-2} deploy/ceph objectstore/bluestore thrashers/default thrashosds-health workloads/small-objects} ----------------------------------------------------------------- time: 00:12:47 info: http://pulpito.ceph.com/kchai-2020-08-22_03:33:19-crimson-rados:thrash-wip-crimson-more-ops-kefu-distro-basic-smithi/5365998/ log: http://qa-proxy.ceph.com/teuthology/kchai-2020-08-22_03:33:19-crimson-rados:thrash-wip-crimson-more-ops-kefu-distro-basic-smithi/5365998/ Command crashed: 'CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --max-ops 400000 --objects 1024 --max-in-flight 64 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 600 --op read 100 --op write 50 --op delete 10 --op write_excl 50 --pool unique_pool_0'