Found 1 jobs Ignoring malparsed backtrace: ceph version 16.0.0-4670-ge307b9aefb4 (e307b9aefb4b0d8735db00b361e12f2d0eda95a2) pacific (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe0) [0x7f6da90ee5dc] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x67d) [0x55acef63eadd] Ignoring malparsed backtrace: 2020-08-25T13:24:18.621 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T13:24:18.621 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":29,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:22:54.339892+0000","last_up_change":"2020-08-25T13:22:53.335338+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":28,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T13:24:18.860+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) Ignoring malparsed backtrace: 2020-08-25T13:25:48.873+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) , 2020-08-25T13:25:50.554 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T13:25:50.834 INFO:teuthology.orchestra.run.smithi144.stdout: Ignoring malparsed backtrace: 2020-08-25T13:27:19.790 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T13:27:19.790 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":29,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:22:54.339892+0000","last_up_change":"2020-08-25T13:22:53.335338+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":28,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T13:27:22.803 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json Ignoring malparsed backtrace: 2020-08-25T13:28:51.769 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T13:28:52.049 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T13:28:52.050 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":29,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:22:54.339892+0000","last_up_change":"2020-08-25T13:22:53.335338+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":28,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} Ignoring malparsed backtrace: 2020-08-25T13:30:20.977 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":29,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:22:54.339892+0000","last_up_change":"2020-08-25T13:22:53.335338+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":28,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T13:30:23.909+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) , 2020-08-25T13:30:23.989 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json Ignoring malparsed backtrace: 2020-08-25T13:31:53.187 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T13:31:53.188 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":29,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:22:54.339892+0000","last_up_change":"2020-08-25T13:22:53.335338+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":28,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T13:31:53.921+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) Ignoring malparsed backtrace: 2020-08-25T13:33:23.933+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) , 2020-08-25T13:33:25.149 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T13:33:25.430 INFO:teuthology.orchestra.run.smithi144.stdout: Ignoring malparsed backtrace: 2020-08-25T13:34:54.385 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T13:34:54.386 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":29,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:22:54.339892+0000","last_up_change":"2020-08-25T13:22:53.335338+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":28,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T13:34:57.399 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json Ignoring malparsed backtrace: 2020-08-25T13:36:26.879 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T13:36:27.157 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T13:36:27.157 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":29,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:22:54.339892+0000","last_up_change":"2020-08-25T13:22:53.335338+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":28,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} Ignoring malparsed backtrace: 2020-08-25T13:37:56.163 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":29,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:22:54.339892+0000","last_up_change":"2020-08-25T13:22:53.335338+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":28,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T13:37:58.973+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) , 2020-08-25T13:37:59.172 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json Ignoring malparsed backtrace: 2020-08-25T13:39:28.217 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T13:39:28.218 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":29,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:22:54.339892+0000","last_up_change":"2020-08-25T13:22:53.335338+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":28,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T13:39:28.983+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) Ignoring malparsed backtrace: 2020-08-25T13:40:58.994+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) , 2020-08-25T13:40:59.951 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T13:41:00.222 INFO:teuthology.orchestra.run.smithi144.stdout: Ignoring malparsed backtrace: 2020-08-25T13:42:28.927 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":29,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:22:54.339892+0000","last_up_change":"2020-08-25T13:22:53.335338+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":28,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T13:42:29.005+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) , 2020-08-25T13:42:31.948 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json Ignoring malparsed backtrace: 2020-08-25T13:44:00.741 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T13:44:01.012 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T13:44:01.012 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":29,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:22:54.339892+0000","last_up_change":"2020-08-25T13:22:53.335338+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":28,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} Ignoring malparsed backtrace: 2020-08-25T13:45:29.810 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":29,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:22:54.339892+0000","last_up_change":"2020-08-25T13:22:53.335338+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":28,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T13:45:32.822 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T13:45:33.101 INFO:teuthology.orchestra.run.smithi144.stdout: Ignoring malparsed backtrace: 2020-08-25T13:47:01.953 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T13:47:01.953 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":29,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:22:54.339892+0000","last_up_change":"2020-08-25T13:22:53.335338+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":28,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T13:47:04.036+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) Ignoring malparsed backtrace: 2020-08-25T13:48:33.759 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T13:48:34.030 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T13:48:34.031 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":29,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:22:54.339892+0000","last_up_change":"2020-08-25T13:22:53.335338+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":28,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} Ignoring malparsed backtrace: 2020-08-25T13:50:02.822 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":29,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:22:54.339892+0000","last_up_change":"2020-08-25T13:22:53.335338+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":28,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T13:50:04.057+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) , 2020-08-25T13:50:05.836 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json Ignoring malparsed backtrace: 2020-08-25T13:51:34.068+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) , 2020-08-25T13:51:34.565 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T13:51:34.835 INFO:teuthology.orchestra.run.smithi144.stdout: Ignoring malparsed backtrace: 2020-08-25T13:53:03.576 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T13:53:04.079+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) , 2020-08-25T13:53:06.588 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json Ignoring malparsed backtrace: 2020-08-25T13:54:35.324 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T13:54:35.618 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T13:54:35.619 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} Ignoring malparsed backtrace: 2020-08-25T13:56:04.365 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T13:56:07.373 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T13:56:07.640 INFO:teuthology.orchestra.run.smithi144.stdout: Ignoring malparsed backtrace: 2020-08-25T13:57:36.354 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T13:57:36.355 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T13:57:39.111+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) Ignoring malparsed backtrace: 2020-08-25T13:59:08.112 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T13:59:08.382 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T13:59:08.383 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} Ignoring malparsed backtrace: 2020-08-25T14:00:37.133 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T14:00:39.132+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) , 2020-08-25T14:00:40.145 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json Ignoring malparsed backtrace: 2020-08-25T14:02:09.143+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) , 2020-08-25T14:02:09.215 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T14:02:09.216 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} Ignoring malparsed backtrace: 2020-08-25T14:03:39.152+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) , 2020-08-25T14:03:40.977 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T14:03:41.254 INFO:teuthology.orchestra.run.smithi144.stdout: Ignoring malparsed backtrace: 2020-08-25T14:05:10.025 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T14:05:10.026 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T14:05:13.038 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json Ignoring malparsed backtrace: 2020-08-25T14:06:41.816 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T14:06:42.086 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T14:06:42.087 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} Ignoring malparsed backtrace: 2020-08-25T14:08:10.899 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T14:08:13.911 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T14:08:14.185+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) Ignoring malparsed backtrace: 2020-08-25T14:09:43.314 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T14:09:43.315 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T14:09:44.196+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) Ignoring malparsed backtrace: 2020-08-25T14:11:14.207+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) , 2020-08-25T14:11:15.087 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T14:11:15.358 INFO:teuthology.orchestra.run.smithi144.stdout: Ignoring malparsed backtrace: 2020-08-25T14:12:44.110 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T14:12:44.217+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) , 2020-08-25T14:12:47.123 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json Ignoring malparsed backtrace: 2020-08-25T14:14:17.790 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T14:14:18.061 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T14:14:18.061 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} Ignoring malparsed backtrace: 2020-08-25T14:15:46.840 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T14:15:49.237+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) , 2020-08-25T14:15:49.853 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json Ignoring malparsed backtrace: 2020-08-25T14:17:18.920 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T14:17:18.921 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T14:17:19.248+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) Ignoring malparsed backtrace: 2020-08-25T14:18:49.259+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) , 2020-08-25T14:18:50.698 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T14:18:50.992 INFO:teuthology.orchestra.run.smithi144.stdout: Ignoring malparsed backtrace: 2020-08-25T14:20:19.730 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T14:20:19.731 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T14:20:22.742 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json Ignoring malparsed backtrace: 2020-08-25T14:21:51.456 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T14:21:51.743 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T14:21:51.743 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} Ignoring malparsed backtrace: 2020-08-25T14:23:20.496 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T14:23:23.508 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T14:23:23.788 INFO:teuthology.orchestra.run.smithi144.stdout: Ignoring malparsed backtrace: 2020-08-25T14:24:52.520 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T14:24:52.520 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T14:24:54.302+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) Ignoring malparsed backtrace: 2020-08-25T14:26:24.275 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T14:26:24.312+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) , 2020-08-25T14:26:24.552 INFO:teuthology.orchestra.run.smithi144.stdout: Ignoring malparsed backtrace: 2020-08-25T14:27:53.380 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T14:27:54.323+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) , 2020-08-25T14:27:56.393 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json Ignoring malparsed backtrace: 2020-08-25T14:29:25.182 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T14:29:25.453 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T14:29:25.454 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} Ignoring malparsed backtrace: 2020-08-25T14:30:54.345+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) , 2020-08-25T14:30:57.333 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T14:30:57.604 INFO:teuthology.orchestra.run.smithi144.stdout: Ignoring malparsed backtrace: 2020-08-25T14:32:26.421 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T14:32:26.422 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T14:32:29.356+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) Ignoring malparsed backtrace: 2020-08-25T14:33:58.234 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T14:33:58.508 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T14:33:58.508 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} Ignoring malparsed backtrace: 2020-08-25T14:35:28.046 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T14:35:29.377+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) , 2020-08-25T14:35:31.059 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json Ignoring malparsed backtrace: 2020-08-25T14:36:59.800 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T14:37:00.069 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T14:37:00.069 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} Ignoring malparsed backtrace: 2020-08-25T14:38:29.399+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) , 2020-08-25T14:38:31.875 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T14:38:32.145 INFO:teuthology.orchestra.run.smithi144.stdout: Ignoring malparsed backtrace: 2020-08-25T14:40:00.955 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T14:40:00.955 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T14:40:03.963 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json Ignoring malparsed backtrace: 2020-08-25T14:41:32.778 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T14:41:33.050 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T14:41:33.051 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} Ignoring malparsed backtrace: 2020-08-25T14:43:01.841 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T14:43:04.431+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) , 2020-08-25T14:43:04.852 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json Ignoring malparsed backtrace: 2020-08-25T14:44:33.902 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T14:44:33.902 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T14:44:34.441+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) Ignoring malparsed backtrace: 2020-08-25T14:46:04.452+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) , 2020-08-25T14:46:05.716 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T14:46:05.990 INFO:teuthology.orchestra.run.smithi144.stdout: Ignoring malparsed backtrace: 2020-08-25T14:47:34.742 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T14:47:34.742 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T14:47:37.755 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json Ignoring malparsed backtrace: 2020-08-25T14:49:07.015 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T14:49:07.286 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T14:49:07.287 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} Ignoring malparsed backtrace: 2020-08-25T14:50:36.115 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T14:50:39.130 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T14:50:39.405 INFO:teuthology.orchestra.run.smithi144.stdout: Ignoring malparsed backtrace: 2020-08-25T14:52:08.143 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T14:52:08.144 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T14:52:09.494+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) Ignoring malparsed backtrace: 2020-08-25T14:53:39.504+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) , 2020-08-25T14:53:39.938 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T14:53:40.214 INFO:teuthology.orchestra.run.smithi144.stdout: Ignoring malparsed backtrace: 2020-08-25T14:55:09.043 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T14:55:09.517+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) , 2020-08-25T14:55:12.060 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json Ignoring malparsed backtrace: 2020-08-25T14:56:41.098 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T14:56:41.368 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T14:56:41.369 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} Ignoring malparsed backtrace: 2020-08-25T14:58:10.140 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T14:58:13.149 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T14:58:13.440 INFO:teuthology.orchestra.run.smithi144.stdout: Ignoring malparsed backtrace: 2020-08-25T14:59:42.529 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T14:59:42.529 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T14:59:44.550+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) Ignoring malparsed backtrace: 2020-08-25T15:01:14.322 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T15:01:14.559+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) , 2020-08-25T15:01:14.609 INFO:teuthology.orchestra.run.smithi144.stdout: Ignoring malparsed backtrace: 2020-08-25T15:02:43.417 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T15:02:44.571+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) , 2020-08-25T15:02:46.442 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json Ignoring malparsed backtrace: 2020-08-25T15:04:15.193 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T15:04:15.488 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T15:04:15.488 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} Ignoring malparsed backtrace: 2020-08-25T15:05:44.591+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) , 2020-08-25T15:05:47.279 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T15:05:47.547 INFO:teuthology.orchestra.run.smithi144.stdout: Ignoring malparsed backtrace: 2020-08-25T15:07:16.335 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T15:07:16.336 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T15:07:19.353 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json Ignoring malparsed backtrace: 2020-08-25T15:08:49.310 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T15:08:49.582 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T15:08:49.583 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} Ignoring malparsed backtrace: 2020-08-25T15:10:18.388 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T15:10:19.621+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) , 2020-08-25T15:10:21.401 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json Ignoring malparsed backtrace: 2020-08-25T15:11:50.176 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T15:11:50.446 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T15:11:50.446 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} Ignoring malparsed backtrace: 2020-08-25T15:13:19.642+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) , 2020-08-25T15:13:22.236 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T15:13:22.508 INFO:teuthology.orchestra.run.smithi144.stdout: Ignoring malparsed backtrace: 2020-08-25T15:14:51.256 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T15:14:51.256 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T15:14:54.269 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json Ignoring malparsed backtrace: 2020-08-25T15:16:23.194 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T15:16:23.466 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T15:16:23.467 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} Ignoring malparsed backtrace: 2020-08-25T15:17:52.198 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T15:17:54.671+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) , 2020-08-25T15:17:55.210 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json Ignoring malparsed backtrace: 2020-08-25T15:19:24.266 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T15:19:24.267 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T15:19:24.682+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) Ignoring malparsed backtrace: 2020-08-25T15:20:54.692+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) , 2020-08-25T15:20:55.985 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T15:20:56.260 INFO:teuthology.orchestra.run.smithi144.stdout: Ignoring malparsed backtrace: 2020-08-25T15:22:25.004 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T15:22:25.004 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T15:22:28.016 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json Ignoring malparsed backtrace: 2020-08-25T15:23:56.812 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T15:23:57.083 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T15:23:57.084 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} Ignoring malparsed backtrace: 2020-08-25T15:25:25.809 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T15:25:28.824 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T15:25:29.096 INFO:teuthology.orchestra.run.smithi144.stdout: Ignoring malparsed backtrace: 2020-08-25T15:26:57.823 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T15:26:57.827 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T15:26:59.733+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) Ignoring malparsed backtrace: 2020-08-25T15:28:29.600 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T15:28:29.743+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) , 2020-08-25T15:28:29.874 INFO:teuthology.orchestra.run.smithi144.stdout: Ignoring malparsed backtrace: 2020-08-25T15:29:58.690 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T15:29:59.753+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) , 2020-08-25T15:30:01.702 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json Ignoring malparsed backtrace: 2020-08-25T15:31:30.483 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T15:31:30.755 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T15:31:30.755 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} Ignoring malparsed backtrace: 2020-08-25T15:32:59.775+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) , 2020-08-25T15:33:02.457 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T15:33:02.728 INFO:teuthology.orchestra.run.smithi144.stdout: Ignoring malparsed backtrace: 2020-08-25T15:34:31.477 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T15:34:31.478 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T15:34:34.490 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json Ignoring malparsed backtrace: 2020-08-25T15:36:03.247 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T15:36:03.520 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T15:36:03.520 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} Ignoring malparsed backtrace: 2020-08-25T15:37:32.223 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T15:37:34.805+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) , 2020-08-25T15:37:35.233 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json Ignoring malparsed backtrace: 2020-08-25T15:39:04.230 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T15:39:04.231 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T15:39:04.817+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) Ignoring malparsed backtrace: 2020-08-25T15:40:34.828+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) , 2020-08-25T15:40:35.958 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T15:40:36.227 INFO:teuthology.orchestra.run.smithi144.stdout: Ignoring malparsed backtrace: 2020-08-25T15:42:04.962 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T15:42:04.963 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T15:42:07.975 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json Ignoring malparsed backtrace: 2020-08-25T15:43:36.695 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T15:43:36.967 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T15:43:36.968 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} Ignoring malparsed backtrace: 2020-08-25T15:45:05.678 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T15:45:08.690 INFO:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early osd dump --format=json , 2020-08-25T15:45:08.964 INFO:teuthology.orchestra.run.smithi144.stdout: Ignoring malparsed backtrace: 2020-08-25T15:46:37.819 INFO:teuthology.orchestra.run.smithi144.stdout: , 2020-08-25T15:46:37.820 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":31,"fsid":"c9dc1fb7-161a-40e4-9961-e6df129c03a9","created":"2020-08-25T13:21:18.871553+0000","modified":"2020-08-25T13:51:20.069588+0000","last_up_change":"2020-08-25T13:51:19.066830+0000","last_in_change":"2020-08-25T13:21:24.224553+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2020-08-25T13:21:22.251459+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2020-08-25T13:21:29.002354+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2020-08-25T13:21:41.514794+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"5b9741c3-2fdd-41ca-9f59-8d9c96275aba","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":26,"down_at":28,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":18864}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":18864}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":18864}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":18864}]},"public_addr":"172.21.15.144:6800/18864","cluster_addr":"172.21.15.144:6801/18864","heartbeat_back_addr":"172.21.15.144:6806/18864","heartbeat_front_addr":"172.21.15.144:6807/18864","state":["exists"]},{"osd":1,"uuid":"ce2dc07b-5c87-4f69-b446-3998c67a4774","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":25,"up_from":26,"up_thru":30,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":18905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":18905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":18905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":18905}]},"public_addr":"172.21.15.144:6802/18905","cluster_addr":"172.21.15.144:6803/18905","heartbeat_back_addr":"172.21.15.144:6804/18905","heartbeat_front_addr":"172.21.15.144:6805/18905","state":["exists","up"]},{"osd":2,"uuid":"9c1d39a9-0448-4305-8918-221ae00665b8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6802","nonce":18706}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6803","nonce":18706}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6806","nonce":18706}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6809","nonce":18706}]},"public_addr":"172.21.15.168:6802/18706","cluster_addr":"172.21.15.168:6803/18706","heartbeat_back_addr":"172.21.15.168:6806/18706","heartbeat_front_addr":"172.21.15.168:6809/18706","state":["exists","up"]},{"osd":3,"uuid":"6cf4d355-e0e7-4982-83b6-32289af4d5c5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":20,"up_from":21,"up_thru":21,"down_at":30,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6804","nonce":18752}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6805","nonce":18752}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6807","nonce":18752}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.168:6808","nonce":18752}]},"public_addr":"172.21.15.168:6804/18752","cluster_addr":"172.21.15.168:6805/18752","heartbeat_back_addr":"172.21.15.168:6807/18752","heartbeat_front_addr":"172.21.15.168:6808/18752","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2020-08-25T13:22:53.335338+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2020-08-25T13:21:53.843837+0000","laggy_probability":0.50999999046325684,"laggy_interval":9,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2020-08-25T13:51:19.066830+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138292837744639,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} , 2020-08-25T15:46:39.870+0000 7ffae77fe700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.2 v2:172.21.15.168:6802/18706 for 24sec e19 v19) Found 1 distinct failure reasons Dead: b'2020-08-25T15:48:04.380 DEBUG:teuthology.exit:Finished running handlers' 1 jobs: ['5375778'] suites: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{default}', 'centos_latest', 'clusters/{fixed-2}', 'crimson-rados:thrash/{0-size-min-size-overrides/3-size-2-min-size', 'deploy/ceph', 'objectstore/bluestore', 'thrashers/default', 'thrashosds-health', 'workloads/small-objects}']