2022-09-08T10:08:41.555 INFO:root:teuthology version: 1.1.1.dev418+gb58f2c18 2022-09-08T10:08:41.557 INFO:teuthology.results:seed: '5255' 2022-09-08T10:09:39.067 INFO:root:teuthology version: 1.1.1.dev418+gb58f2c18 2022-09-08T10:09:39.072 INFO:teuthology.results:Waiting up to 43200 seconds for tests to finish... 2022-09-08T22:10:34.643 WARNING:teuthology.results:test(s) did not finish before timeout of 43200 seconds 2022-09-08T22:10:34.937 INFO:scrape:Found 3 jobs 2022-09-08T22:10:35.093 WARNING:scrape:Ignoring malparsed backtrace: ceph version 17.0.0-14639-g24829a3e (24829a3e79a41a6050b0aa4e3866b596e91b3ad5) quincy (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7fe561b53cb0] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x22e) [0x560d8a3345a8] 2022-09-08T22:10:35.094 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T10:26:02.103 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet , 2022-09-08T10:26:02.104 INFO:tasks.ceph.ceph_manager.ceph:waiting for all up , 2022-09-08T10:26:02.104 DEBUG:teuthology.orchestra.run.smithi074:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2022-09-08T22:10:35.110 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T10:27:33.067 INFO:teuthology.orchestra.run.smithi074.stdout: , 2022-09-08T10:27:33.068 INFO:teuthology.orchestra.run.smithi074.stdout:{"epoch":59,"fsid":"bf7c7e43-6d34-4a0d-9498-2ac81ecf5b2c","created":"2022-09-08T10:24:06.958714+0000","modified":"2022-09-08T10:26:01.098270+0000","last_up_change":"2022-09-08T10:26:00.093462+0000","last_in_change":"2022-09-08T10:24:08.200859+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-08T10:24:14.537920+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-09-08T10:24:16.463971+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-09-08T10:24:23.404862+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"0fda8af2-85b4-4d23-8fbf-1ef0a608b9bc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6800","nonce":38348}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6801","nonce":38348}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6804","nonce":38348}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6805","nonce":38348}]},"public_addr":"172.21.15.74:6800/38348","cluster_addr":"172.21.15.74:6801/38348","heartbeat_back_addr":"172.21.15.74:6804/38348","heartbeat_front_addr":"172.21.15.74:6805/38348","state":["exists","up"]},{"osd":1,"uuid":"bd886954-560e-4fcb-aaf0-39253aabe76b","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":15,"down_at":45,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6802","nonce":38375}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6803","nonce":38375}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6806","nonce":38375}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6807","nonce":38375}]},"public_addr":"172.21.15.74:6802/38375","cluster_addr":"172.21.15.74:6803/38375","heartbeat_back_addr":"172.21.15.74:6806/38375","heartbeat_front_addr":"172.21.15.74:6807/38375","state":["exists"]},{"osd":2,"uuid":"a7e4874b-49d1-4432-96ff-966d4aff6234","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6801","nonce":38010}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6800","nonce":38010}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6804","nonce":38010}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6805","nonce":38010}]},"public_addr":"172.21.15.114:6801/38010","cluster_addr":"172.21.15.114:6800/38010","heartbeat_back_addr":"172.21.15.114:6804/38010","heartbeat_front_addr":"172.21.15.114:6805/38010","state":["exists","up"]},{"osd":3,"uuid":"12a13c84-8a2a-4550-b795-e91cc02c4e8e","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":45,"down_at":58,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6802","nonce":38037}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6803","nonce":38037}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6806","nonce":38037}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6807","nonce":38037}]},"public_addr":"172.21.15.114:6802/38037","cluster_addr":"172.21.15.114:6803/38037","heartbeat_back_addr":"172.21.15.114:6806/38037","heartbeat_front_addr":"172.21.15.114:6807/38037","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2022-09-08T10:25:29.996083+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2022-09-08T10:26:00.093462+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2022-09-08T10:27:36.083 DEBUG:teuthology.orchestra.run.smithi074:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2022-09-08T22:10:35.143 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T10:29:07.082 INFO:teuthology.orchestra.run.smithi074.stdout: , 2022-09-08T10:29:07.083 INFO:teuthology.orchestra.run.smithi074.stdout:{"epoch":59,"fsid":"bf7c7e43-6d34-4a0d-9498-2ac81ecf5b2c","created":"2022-09-08T10:24:06.958714+0000","modified":"2022-09-08T10:26:01.098270+0000","last_up_change":"2022-09-08T10:26:00.093462+0000","last_in_change":"2022-09-08T10:24:08.200859+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-08T10:24:14.537920+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-09-08T10:24:16.463971+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-09-08T10:24:23.404862+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"0fda8af2-85b4-4d23-8fbf-1ef0a608b9bc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6800","nonce":38348}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6801","nonce":38348}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6804","nonce":38348}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6805","nonce":38348}]},"public_addr":"172.21.15.74:6800/38348","cluster_addr":"172.21.15.74:6801/38348","heartbeat_back_addr":"172.21.15.74:6804/38348","heartbeat_front_addr":"172.21.15.74:6805/38348","state":["exists","up"]},{"osd":1,"uuid":"bd886954-560e-4fcb-aaf0-39253aabe76b","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":15,"down_at":45,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6802","nonce":38375}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6803","nonce":38375}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6806","nonce":38375}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6807","nonce":38375}]},"public_addr":"172.21.15.74:6802/38375","cluster_addr":"172.21.15.74:6803/38375","heartbeat_back_addr":"172.21.15.74:6806/38375","heartbeat_front_addr":"172.21.15.74:6807/38375","state":["exists"]},{"osd":2,"uuid":"a7e4874b-49d1-4432-96ff-966d4aff6234","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6801","nonce":38010}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6800","nonce":38010}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6804","nonce":38010}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6805","nonce":38010}]},"public_addr":"172.21.15.114:6801/38010","cluster_addr":"172.21.15.114:6800/38010","heartbeat_back_addr":"172.21.15.114:6804/38010","heartbeat_front_addr":"172.21.15.114:6805/38010","state":["exists","up"]},{"osd":3,"uuid":"12a13c84-8a2a-4550-b795-e91cc02c4e8e","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":45,"down_at":58,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6802","nonce":38037}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6803","nonce":38037}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6806","nonce":38037}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6807","nonce":38037}]},"public_addr":"172.21.15.114:6802/38037","cluster_addr":"172.21.15.114:6803/38037","heartbeat_back_addr":"172.21.15.114:6806/38037","heartbeat_front_addr":"172.21.15.114:6807/38037","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2022-09-08T10:25:29.996083+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2022-09-08T10:26:00.093462+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2022-09-08T10:29:10.098 DEBUG:teuthology.orchestra.run.smithi074:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2022-09-08T22:10:35.144 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T10:30:40.911 INFO:teuthology.orchestra.run.smithi074.stdout: , 2022-09-08T10:30:40.912 INFO:teuthology.orchestra.run.smithi074.stdout:{"epoch":59,"fsid":"bf7c7e43-6d34-4a0d-9498-2ac81ecf5b2c","created":"2022-09-08T10:24:06.958714+0000","modified":"2022-09-08T10:26:01.098270+0000","last_up_change":"2022-09-08T10:26:00.093462+0000","last_in_change":"2022-09-08T10:24:08.200859+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-08T10:24:14.537920+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-09-08T10:24:16.463971+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-09-08T10:24:23.404862+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"0fda8af2-85b4-4d23-8fbf-1ef0a608b9bc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6800","nonce":38348}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6801","nonce":38348}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6804","nonce":38348}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6805","nonce":38348}]},"public_addr":"172.21.15.74:6800/38348","cluster_addr":"172.21.15.74:6801/38348","heartbeat_back_addr":"172.21.15.74:6804/38348","heartbeat_front_addr":"172.21.15.74:6805/38348","state":["exists","up"]},{"osd":1,"uuid":"bd886954-560e-4fcb-aaf0-39253aabe76b","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":15,"down_at":45,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6802","nonce":38375}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6803","nonce":38375}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6806","nonce":38375}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6807","nonce":38375}]},"public_addr":"172.21.15.74:6802/38375","cluster_addr":"172.21.15.74:6803/38375","heartbeat_back_addr":"172.21.15.74:6806/38375","heartbeat_front_addr":"172.21.15.74:6807/38375","state":["exists"]},{"osd":2,"uuid":"a7e4874b-49d1-4432-96ff-966d4aff6234","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6801","nonce":38010}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6800","nonce":38010}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6804","nonce":38010}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6805","nonce":38010}]},"public_addr":"172.21.15.114:6801/38010","cluster_addr":"172.21.15.114:6800/38010","heartbeat_back_addr":"172.21.15.114:6804/38010","heartbeat_front_addr":"172.21.15.114:6805/38010","state":["exists","up"]},{"osd":3,"uuid":"12a13c84-8a2a-4550-b795-e91cc02c4e8e","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":45,"down_at":58,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6802","nonce":38037}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6803","nonce":38037}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6806","nonce":38037}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6807","nonce":38037}]},"public_addr":"172.21.15.114:6802/38037","cluster_addr":"172.21.15.114:6803/38037","heartbeat_back_addr":"172.21.15.114:6806/38037","heartbeat_front_addr":"172.21.15.114:6807/38037","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2022-09-08T10:25:29.996083+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2022-09-08T10:26:00.093462+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2022-09-08T10:30:43.927 DEBUG:teuthology.orchestra.run.smithi074:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2022-09-08T22:10:35.145 WARNING:scrape:Ignoring malparsed backtrace: File "/usr/lib/python3.6/contextlib.py", line 88, in __exit__ , next(self.gen) , File "/home/teuthworker/src/github.com_Matan-B_ceph_30e344a9ce106774ae58dd1803482b9c71d152ad/qa/tasks/rados.py", line 286, in task 2022-09-08T22:10:35.146 WARNING:scrape:Ignoring malparsed backtrace: , Traceback (most recent call last): , File "/home/teuthworker/src/git.ceph.com_git_teuthology_b58f2c18636eb10faa77ed3614abd00cb85dfc2c/teuthology/orchestra/daemon/state.py", line 139, in stop 2022-09-08T22:10:35.147 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T10:35:04.551 DEBUG:teuthology.orchestra.run.smithi114:> sync && sudo umount -f /var/lib/ceph/osd/ceph-2 , 2022-09-08T10:35:04.647 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-3 on ubuntu@smithi114.front.sepia.ceph.com , 2022-09-08T10:35:04.648 DEBUG:teuthology.orchestra.run.smithi114:> sync && sudo umount -f /var/lib/ceph/osd/ceph-3 2022-09-08T22:10:35.148 WARNING:scrape:Ignoring malparsed backtrace: File "/home/teuthworker/src/github.com_Matan-B_ceph_30e344a9ce106774ae58dd1803482b9c71d152ad/qa/tasks/rados.py", line 274, in thread , run.wait(tests.values()) , File "/home/teuthworker/src/git.ceph.com_git_teuthology_b58f2c18636eb10faa77ed3614abd00cb85dfc2c/teuthology/orchestra/run.py", line 479, in wait 2022-09-08T22:10:35.149 WARNING:scrape:Ignoring malparsed backtrace: next(self.gen) , File "/home/teuthworker/src/github.com_Matan-B_ceph_30e344a9ce106774ae58dd1803482b9c71d152ad/qa/tasks/rados.py", line 286, in task , running.get() 2022-09-08T22:10:35.176 WARNING:scrape:Ignoring malparsed backtrace: ceph version 17.0.0-14639-g24829a3e (24829a3e79a41a6050b0aa4e3866b596e91b3ad5) quincy (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7fe561b53cb0] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x22e) [0x560d8a3345a8] 2022-09-08T22:10:35.177 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T10:26:02.103 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet , 2022-09-08T10:26:02.104 INFO:tasks.ceph.ceph_manager.ceph:waiting for all up , 2022-09-08T10:26:02.104 DEBUG:teuthology.orchestra.run.smithi074:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2022-09-08T22:10:35.178 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T10:27:33.067 INFO:teuthology.orchestra.run.smithi074.stdout: , 2022-09-08T10:27:33.068 INFO:teuthology.orchestra.run.smithi074.stdout:{"epoch":59,"fsid":"bf7c7e43-6d34-4a0d-9498-2ac81ecf5b2c","created":"2022-09-08T10:24:06.958714+0000","modified":"2022-09-08T10:26:01.098270+0000","last_up_change":"2022-09-08T10:26:00.093462+0000","last_in_change":"2022-09-08T10:24:08.200859+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-08T10:24:14.537920+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-09-08T10:24:16.463971+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-09-08T10:24:23.404862+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"0fda8af2-85b4-4d23-8fbf-1ef0a608b9bc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6800","nonce":38348}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6801","nonce":38348}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6804","nonce":38348}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6805","nonce":38348}]},"public_addr":"172.21.15.74:6800/38348","cluster_addr":"172.21.15.74:6801/38348","heartbeat_back_addr":"172.21.15.74:6804/38348","heartbeat_front_addr":"172.21.15.74:6805/38348","state":["exists","up"]},{"osd":1,"uuid":"bd886954-560e-4fcb-aaf0-39253aabe76b","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":15,"down_at":45,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6802","nonce":38375}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6803","nonce":38375}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6806","nonce":38375}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6807","nonce":38375}]},"public_addr":"172.21.15.74:6802/38375","cluster_addr":"172.21.15.74:6803/38375","heartbeat_back_addr":"172.21.15.74:6806/38375","heartbeat_front_addr":"172.21.15.74:6807/38375","state":["exists"]},{"osd":2,"uuid":"a7e4874b-49d1-4432-96ff-966d4aff6234","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6801","nonce":38010}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6800","nonce":38010}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6804","nonce":38010}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6805","nonce":38010}]},"public_addr":"172.21.15.114:6801/38010","cluster_addr":"172.21.15.114:6800/38010","heartbeat_back_addr":"172.21.15.114:6804/38010","heartbeat_front_addr":"172.21.15.114:6805/38010","state":["exists","up"]},{"osd":3,"uuid":"12a13c84-8a2a-4550-b795-e91cc02c4e8e","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":45,"down_at":58,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6802","nonce":38037}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6803","nonce":38037}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6806","nonce":38037}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6807","nonce":38037}]},"public_addr":"172.21.15.114:6802/38037","cluster_addr":"172.21.15.114:6803/38037","heartbeat_back_addr":"172.21.15.114:6806/38037","heartbeat_front_addr":"172.21.15.114:6807/38037","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2022-09-08T10:25:29.996083+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2022-09-08T10:26:00.093462+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2022-09-08T10:27:36.083 DEBUG:teuthology.orchestra.run.smithi074:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2022-09-08T22:10:35.179 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T10:29:07.082 INFO:teuthology.orchestra.run.smithi074.stdout: , 2022-09-08T10:29:07.083 INFO:teuthology.orchestra.run.smithi074.stdout:{"epoch":59,"fsid":"bf7c7e43-6d34-4a0d-9498-2ac81ecf5b2c","created":"2022-09-08T10:24:06.958714+0000","modified":"2022-09-08T10:26:01.098270+0000","last_up_change":"2022-09-08T10:26:00.093462+0000","last_in_change":"2022-09-08T10:24:08.200859+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-08T10:24:14.537920+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-09-08T10:24:16.463971+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-09-08T10:24:23.404862+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"0fda8af2-85b4-4d23-8fbf-1ef0a608b9bc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6800","nonce":38348}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6801","nonce":38348}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6804","nonce":38348}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6805","nonce":38348}]},"public_addr":"172.21.15.74:6800/38348","cluster_addr":"172.21.15.74:6801/38348","heartbeat_back_addr":"172.21.15.74:6804/38348","heartbeat_front_addr":"172.21.15.74:6805/38348","state":["exists","up"]},{"osd":1,"uuid":"bd886954-560e-4fcb-aaf0-39253aabe76b","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":15,"down_at":45,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6802","nonce":38375}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6803","nonce":38375}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6806","nonce":38375}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6807","nonce":38375}]},"public_addr":"172.21.15.74:6802/38375","cluster_addr":"172.21.15.74:6803/38375","heartbeat_back_addr":"172.21.15.74:6806/38375","heartbeat_front_addr":"172.21.15.74:6807/38375","state":["exists"]},{"osd":2,"uuid":"a7e4874b-49d1-4432-96ff-966d4aff6234","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6801","nonce":38010}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6800","nonce":38010}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6804","nonce":38010}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6805","nonce":38010}]},"public_addr":"172.21.15.114:6801/38010","cluster_addr":"172.21.15.114:6800/38010","heartbeat_back_addr":"172.21.15.114:6804/38010","heartbeat_front_addr":"172.21.15.114:6805/38010","state":["exists","up"]},{"osd":3,"uuid":"12a13c84-8a2a-4550-b795-e91cc02c4e8e","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":45,"down_at":58,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6802","nonce":38037}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6803","nonce":38037}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6806","nonce":38037}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6807","nonce":38037}]},"public_addr":"172.21.15.114:6802/38037","cluster_addr":"172.21.15.114:6803/38037","heartbeat_back_addr":"172.21.15.114:6806/38037","heartbeat_front_addr":"172.21.15.114:6807/38037","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2022-09-08T10:25:29.996083+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2022-09-08T10:26:00.093462+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2022-09-08T10:29:10.098 DEBUG:teuthology.orchestra.run.smithi074:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2022-09-08T22:10:35.180 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T10:30:40.911 INFO:teuthology.orchestra.run.smithi074.stdout: , 2022-09-08T10:30:40.912 INFO:teuthology.orchestra.run.smithi074.stdout:{"epoch":59,"fsid":"bf7c7e43-6d34-4a0d-9498-2ac81ecf5b2c","created":"2022-09-08T10:24:06.958714+0000","modified":"2022-09-08T10:26:01.098270+0000","last_up_change":"2022-09-08T10:26:00.093462+0000","last_in_change":"2022-09-08T10:24:08.200859+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-08T10:24:14.537920+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-09-08T10:24:16.463971+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-09-08T10:24:23.404862+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"0fda8af2-85b4-4d23-8fbf-1ef0a608b9bc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6800","nonce":38348}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6801","nonce":38348}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6804","nonce":38348}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6805","nonce":38348}]},"public_addr":"172.21.15.74:6800/38348","cluster_addr":"172.21.15.74:6801/38348","heartbeat_back_addr":"172.21.15.74:6804/38348","heartbeat_front_addr":"172.21.15.74:6805/38348","state":["exists","up"]},{"osd":1,"uuid":"bd886954-560e-4fcb-aaf0-39253aabe76b","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":15,"down_at":45,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6802","nonce":38375}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6803","nonce":38375}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6806","nonce":38375}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6807","nonce":38375}]},"public_addr":"172.21.15.74:6802/38375","cluster_addr":"172.21.15.74:6803/38375","heartbeat_back_addr":"172.21.15.74:6806/38375","heartbeat_front_addr":"172.21.15.74:6807/38375","state":["exists"]},{"osd":2,"uuid":"a7e4874b-49d1-4432-96ff-966d4aff6234","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6801","nonce":38010}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6800","nonce":38010}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6804","nonce":38010}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6805","nonce":38010}]},"public_addr":"172.21.15.114:6801/38010","cluster_addr":"172.21.15.114:6800/38010","heartbeat_back_addr":"172.21.15.114:6804/38010","heartbeat_front_addr":"172.21.15.114:6805/38010","state":["exists","up"]},{"osd":3,"uuid":"12a13c84-8a2a-4550-b795-e91cc02c4e8e","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":45,"down_at":58,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6802","nonce":38037}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6803","nonce":38037}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6806","nonce":38037}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6807","nonce":38037}]},"public_addr":"172.21.15.114:6802/38037","cluster_addr":"172.21.15.114:6803/38037","heartbeat_back_addr":"172.21.15.114:6806/38037","heartbeat_front_addr":"172.21.15.114:6807/38037","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2022-09-08T10:25:29.996083+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2022-09-08T10:26:00.093462+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2022-09-08T10:30:43.927 DEBUG:teuthology.orchestra.run.smithi074:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2022-09-08T22:10:35.181 WARNING:scrape:Ignoring malparsed backtrace: File "/usr/lib/python3.6/contextlib.py", line 88, in __exit__ , next(self.gen) , File "/home/teuthworker/src/github.com_Matan-B_ceph_30e344a9ce106774ae58dd1803482b9c71d152ad/qa/tasks/rados.py", line 286, in task 2022-09-08T22:10:35.182 WARNING:scrape:Ignoring malparsed backtrace: , Traceback (most recent call last): , File "/home/teuthworker/src/git.ceph.com_git_teuthology_b58f2c18636eb10faa77ed3614abd00cb85dfc2c/teuthology/orchestra/daemon/state.py", line 139, in stop 2022-09-08T22:10:35.182 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T10:35:04.551 DEBUG:teuthology.orchestra.run.smithi114:> sync && sudo umount -f /var/lib/ceph/osd/ceph-2 , 2022-09-08T10:35:04.647 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-3 on ubuntu@smithi114.front.sepia.ceph.com , 2022-09-08T10:35:04.648 DEBUG:teuthology.orchestra.run.smithi114:> sync && sudo umount -f /var/lib/ceph/osd/ceph-3 2022-09-08T22:10:35.183 WARNING:scrape:Ignoring malparsed backtrace: File "/home/teuthworker/src/github.com_Matan-B_ceph_30e344a9ce106774ae58dd1803482b9c71d152ad/qa/tasks/rados.py", line 274, in thread , run.wait(tests.values()) , File "/home/teuthworker/src/git.ceph.com_git_teuthology_b58f2c18636eb10faa77ed3614abd00cb85dfc2c/teuthology/orchestra/run.py", line 479, in wait 2022-09-08T22:10:35.184 WARNING:scrape:Ignoring malparsed backtrace: next(self.gen) , File "/home/teuthworker/src/github.com_Matan-B_ceph_30e344a9ce106774ae58dd1803482b9c71d152ad/qa/tasks/rados.py", line 286, in task , running.get() 2022-09-08T22:10:35.209 WARNING:scrape:Ignoring malparsed backtrace: ceph version 17.0.0-14639-g24829a3e (24829a3e79a41a6050b0aa4e3866b596e91b3ad5) quincy (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7fe561b53cb0] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x22e) [0x560d8a3345a8] 2022-09-08T22:10:35.211 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T10:26:02.103 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet , 2022-09-08T10:26:02.104 INFO:tasks.ceph.ceph_manager.ceph:waiting for all up , 2022-09-08T10:26:02.104 DEBUG:teuthology.orchestra.run.smithi074:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2022-09-08T22:10:35.212 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T10:27:33.067 INFO:teuthology.orchestra.run.smithi074.stdout: , 2022-09-08T10:27:33.068 INFO:teuthology.orchestra.run.smithi074.stdout:{"epoch":59,"fsid":"bf7c7e43-6d34-4a0d-9498-2ac81ecf5b2c","created":"2022-09-08T10:24:06.958714+0000","modified":"2022-09-08T10:26:01.098270+0000","last_up_change":"2022-09-08T10:26:00.093462+0000","last_in_change":"2022-09-08T10:24:08.200859+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-08T10:24:14.537920+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-09-08T10:24:16.463971+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-09-08T10:24:23.404862+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"0fda8af2-85b4-4d23-8fbf-1ef0a608b9bc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6800","nonce":38348}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6801","nonce":38348}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6804","nonce":38348}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6805","nonce":38348}]},"public_addr":"172.21.15.74:6800/38348","cluster_addr":"172.21.15.74:6801/38348","heartbeat_back_addr":"172.21.15.74:6804/38348","heartbeat_front_addr":"172.21.15.74:6805/38348","state":["exists","up"]},{"osd":1,"uuid":"bd886954-560e-4fcb-aaf0-39253aabe76b","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":15,"down_at":45,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6802","nonce":38375}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6803","nonce":38375}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6806","nonce":38375}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6807","nonce":38375}]},"public_addr":"172.21.15.74:6802/38375","cluster_addr":"172.21.15.74:6803/38375","heartbeat_back_addr":"172.21.15.74:6806/38375","heartbeat_front_addr":"172.21.15.74:6807/38375","state":["exists"]},{"osd":2,"uuid":"a7e4874b-49d1-4432-96ff-966d4aff6234","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6801","nonce":38010}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6800","nonce":38010}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6804","nonce":38010}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6805","nonce":38010}]},"public_addr":"172.21.15.114:6801/38010","cluster_addr":"172.21.15.114:6800/38010","heartbeat_back_addr":"172.21.15.114:6804/38010","heartbeat_front_addr":"172.21.15.114:6805/38010","state":["exists","up"]},{"osd":3,"uuid":"12a13c84-8a2a-4550-b795-e91cc02c4e8e","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":45,"down_at":58,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6802","nonce":38037}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6803","nonce":38037}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6806","nonce":38037}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6807","nonce":38037}]},"public_addr":"172.21.15.114:6802/38037","cluster_addr":"172.21.15.114:6803/38037","heartbeat_back_addr":"172.21.15.114:6806/38037","heartbeat_front_addr":"172.21.15.114:6807/38037","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2022-09-08T10:25:29.996083+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2022-09-08T10:26:00.093462+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2022-09-08T10:27:36.083 DEBUG:teuthology.orchestra.run.smithi074:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2022-09-08T22:10:35.213 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T10:29:07.082 INFO:teuthology.orchestra.run.smithi074.stdout: , 2022-09-08T10:29:07.083 INFO:teuthology.orchestra.run.smithi074.stdout:{"epoch":59,"fsid":"bf7c7e43-6d34-4a0d-9498-2ac81ecf5b2c","created":"2022-09-08T10:24:06.958714+0000","modified":"2022-09-08T10:26:01.098270+0000","last_up_change":"2022-09-08T10:26:00.093462+0000","last_in_change":"2022-09-08T10:24:08.200859+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-08T10:24:14.537920+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-09-08T10:24:16.463971+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-09-08T10:24:23.404862+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"0fda8af2-85b4-4d23-8fbf-1ef0a608b9bc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6800","nonce":38348}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6801","nonce":38348}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6804","nonce":38348}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6805","nonce":38348}]},"public_addr":"172.21.15.74:6800/38348","cluster_addr":"172.21.15.74:6801/38348","heartbeat_back_addr":"172.21.15.74:6804/38348","heartbeat_front_addr":"172.21.15.74:6805/38348","state":["exists","up"]},{"osd":1,"uuid":"bd886954-560e-4fcb-aaf0-39253aabe76b","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":15,"down_at":45,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6802","nonce":38375}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6803","nonce":38375}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6806","nonce":38375}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6807","nonce":38375}]},"public_addr":"172.21.15.74:6802/38375","cluster_addr":"172.21.15.74:6803/38375","heartbeat_back_addr":"172.21.15.74:6806/38375","heartbeat_front_addr":"172.21.15.74:6807/38375","state":["exists"]},{"osd":2,"uuid":"a7e4874b-49d1-4432-96ff-966d4aff6234","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6801","nonce":38010}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6800","nonce":38010}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6804","nonce":38010}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6805","nonce":38010}]},"public_addr":"172.21.15.114:6801/38010","cluster_addr":"172.21.15.114:6800/38010","heartbeat_back_addr":"172.21.15.114:6804/38010","heartbeat_front_addr":"172.21.15.114:6805/38010","state":["exists","up"]},{"osd":3,"uuid":"12a13c84-8a2a-4550-b795-e91cc02c4e8e","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":45,"down_at":58,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6802","nonce":38037}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6803","nonce":38037}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6806","nonce":38037}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6807","nonce":38037}]},"public_addr":"172.21.15.114:6802/38037","cluster_addr":"172.21.15.114:6803/38037","heartbeat_back_addr":"172.21.15.114:6806/38037","heartbeat_front_addr":"172.21.15.114:6807/38037","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2022-09-08T10:25:29.996083+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2022-09-08T10:26:00.093462+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2022-09-08T10:29:10.098 DEBUG:teuthology.orchestra.run.smithi074:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2022-09-08T22:10:35.214 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T10:30:40.911 INFO:teuthology.orchestra.run.smithi074.stdout: , 2022-09-08T10:30:40.912 INFO:teuthology.orchestra.run.smithi074.stdout:{"epoch":59,"fsid":"bf7c7e43-6d34-4a0d-9498-2ac81ecf5b2c","created":"2022-09-08T10:24:06.958714+0000","modified":"2022-09-08T10:26:01.098270+0000","last_up_change":"2022-09-08T10:26:00.093462+0000","last_in_change":"2022-09-08T10:24:08.200859+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-08T10:24:14.537920+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-09-08T10:24:16.463971+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-09-08T10:24:23.404862+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"0fda8af2-85b4-4d23-8fbf-1ef0a608b9bc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6800","nonce":38348}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6801","nonce":38348}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6804","nonce":38348}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6805","nonce":38348}]},"public_addr":"172.21.15.74:6800/38348","cluster_addr":"172.21.15.74:6801/38348","heartbeat_back_addr":"172.21.15.74:6804/38348","heartbeat_front_addr":"172.21.15.74:6805/38348","state":["exists","up"]},{"osd":1,"uuid":"bd886954-560e-4fcb-aaf0-39253aabe76b","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":15,"down_at":45,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6802","nonce":38375}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6803","nonce":38375}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6806","nonce":38375}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.74:6807","nonce":38375}]},"public_addr":"172.21.15.74:6802/38375","cluster_addr":"172.21.15.74:6803/38375","heartbeat_back_addr":"172.21.15.74:6806/38375","heartbeat_front_addr":"172.21.15.74:6807/38375","state":["exists"]},{"osd":2,"uuid":"a7e4874b-49d1-4432-96ff-966d4aff6234","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6801","nonce":38010}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6800","nonce":38010}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6804","nonce":38010}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6805","nonce":38010}]},"public_addr":"172.21.15.114:6801/38010","cluster_addr":"172.21.15.114:6800/38010","heartbeat_back_addr":"172.21.15.114:6804/38010","heartbeat_front_addr":"172.21.15.114:6805/38010","state":["exists","up"]},{"osd":3,"uuid":"12a13c84-8a2a-4550-b795-e91cc02c4e8e","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":45,"down_at":58,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6802","nonce":38037}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6803","nonce":38037}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6806","nonce":38037}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:6807","nonce":38037}]},"public_addr":"172.21.15.114:6802/38037","cluster_addr":"172.21.15.114:6803/38037","heartbeat_back_addr":"172.21.15.114:6806/38037","heartbeat_front_addr":"172.21.15.114:6807/38037","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2022-09-08T10:25:29.996083+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2022-09-08T10:26:00.093462+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2022-09-08T10:30:43.927 DEBUG:teuthology.orchestra.run.smithi074:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2022-09-08T22:10:35.215 WARNING:scrape:Ignoring malparsed backtrace: File "/usr/lib/python3.6/contextlib.py", line 88, in __exit__ , next(self.gen) , File "/home/teuthworker/src/github.com_Matan-B_ceph_30e344a9ce106774ae58dd1803482b9c71d152ad/qa/tasks/rados.py", line 286, in task 2022-09-08T22:10:35.217 WARNING:scrape:Ignoring malparsed backtrace: , Traceback (most recent call last): , File "/home/teuthworker/src/git.ceph.com_git_teuthology_b58f2c18636eb10faa77ed3614abd00cb85dfc2c/teuthology/orchestra/daemon/state.py", line 139, in stop 2022-09-08T22:10:35.218 WARNING:scrape:Ignoring malparsed backtrace: 2022-09-08T10:35:04.551 DEBUG:teuthology.orchestra.run.smithi114:> sync && sudo umount -f /var/lib/ceph/osd/ceph-2 , 2022-09-08T10:35:04.647 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-3 on ubuntu@smithi114.front.sepia.ceph.com , 2022-09-08T10:35:04.648 DEBUG:teuthology.orchestra.run.smithi114:> sync && sudo umount -f /var/lib/ceph/osd/ceph-3 2022-09-08T22:10:35.219 WARNING:scrape:Ignoring malparsed backtrace: File "/home/teuthworker/src/github.com_Matan-B_ceph_30e344a9ce106774ae58dd1803482b9c71d152ad/qa/tasks/rados.py", line 274, in thread , run.wait(tests.values()) , File "/home/teuthworker/src/git.ceph.com_git_teuthology_b58f2c18636eb10faa77ed3614abd00cb85dfc2c/teuthology/orchestra/run.py", line 479, in wait 2022-09-08T22:10:35.220 WARNING:scrape:Ignoring malparsed backtrace: next(self.gen) , File "/home/teuthworker/src/github.com_Matan-B_ceph_30e344a9ce106774ae58dd1803482b9c71d152ad/qa/tasks/rados.py", line 286, in task , running.get() 2022-09-08T22:10:35.335 INFO:scrape:Found 2 distinct failure reasons 2022-09-08T22:10:35.336 INFO:scrape:Failure: Command crashed: 'CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --max-ops 400000 --objects 1024 --max-in-flight 64 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 600 --op read 100 --op write 50 --op delete 10 --op write_excl 50 --pool unique_pool_0' 2022-09-08T22:10:35.337 INFO:scrape:1 jobs: ['7020180'] 2022-09-08T22:10:35.338 INFO:scrape:suites: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{default}', 'centos_latest', 'clusters/{fixed-2}', 'crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'deploy/ceph', 'objectstore/bluestore', 'thrashers/default', 'thrashosds-health', 'workloads/small-objects}'] 2022-09-08T22:10:35.338 INFO:scrape: 2022-09-08T22:10:35.339 INFO:scrape:Dead: b'2022-09-08T22:10:34.516 INFO:teuthology.orchestra.run.smithi170.stdout:{"epoch":54,"fsid":"81dab200-ab87-4ac3-bb2e-1ce0f57b2630","created":"2022-09-08T10:23:37.994978+0000","modified":"2022-09-08T10:34:58.088885+0000","last_up_change":"2022-09-08T10:24:55.058606+0000","last_in_change":"2022-09-08T10:34:58.088885+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-08T10:23:45.662632+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0\'0","target_version":"0\'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-09-08T10:23:47.493303+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0\'0","target_version":"0\'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-09-08T10:24:00.004509+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0\'0","target_version":"0\'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"9b24e188-ba38-4652-8c61-72acf5675e57","up":0,"in":0,"weight":0,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":15,"down_at":31,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6800","nonce":38361}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6801","nonce":38361}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6805","nonce":38361}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6804","nonce":38361}]},"public_addr":"172.21.15.170:6800/38361","cluster_addr":"172.21.15.170:6801/38361","heartbeat_back_addr":"172.21.15.170:6805/38361","heartbeat_front_addr":"172.21.15.170:6804/38361","state":["autoout","exists"]},{"osd":1,"uuid":"b4798e2e-87ac-4387-8d05-29db2de29c15","up":0,"in":0,"weight":0,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":15,"down_at":37,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6803","nonce":38388}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6802","nonce":38388}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6806","nonce":38388}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6807","nonce":38388}]},"public_addr":"172.21.15.170:6803/38388","cluster_addr":"172.21.15.170:6802/38388","heartbeat_back_addr":"172.21.15.170:6806/38388","heartbeat_front_addr":"172.21.15.170:6807/38388","state":["autoout","exists"]},{"osd":2,"uuid":"879f613d-18df-479b-ae81-4b14cfd7ffbc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6800","nonce":38079}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6801","nonce":38079}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6804","nonce":38079}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6805","nonce":38079}]},"public_addr":"172.21.15.189:6800/38079","cluster_addr":"172.21.15.189:6801/38079","heartbeat_back_addr":"172.21.15.189:6804/38079","heartbeat_front_addr":"172.21.15.189:6805/38079","state":["exists","up"]},{"osd":3,"uuid":"f17a1187-c049-46e4-a9b3-ce71b442049e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6803","nonce":38108}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6802","nonce":38108}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6806","nonce":38108}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6807","nonce":38108}]},"public_addr":"172.21.15.189:6803/38108","cluster_addr":"172.21.15.189:6802/38108","heartbeat_back_addr":"172.21.15.189:6806/38108","heartbeat_front_addr":"172.21.15.189:6807/38108","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2022-09-08T10:24:43.032417+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":65536,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2022-09-08T10:24:55.058606+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":65536,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.2","mappings":[{"from":2,"to":3}]},{"pgid":"2.3","mappings":[{"from":2,"to":3}]},{"pgid":"3.d","mappings":[{"from":3,"to":2}]}],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}}' 2022-09-08T22:10:35.339 INFO:scrape:2 jobs: ['7020179', '7020181'] 2022-09-08T22:10:35.340 INFO:scrape:suites intersection: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{default}', 'centos_latest', 'clusters/{fixed-2}', 'crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'deploy/ceph', 'objectstore/bluestore', 'thrashers/default', 'thrashosds-health', 'workloads/small-objects}'] 2022-09-08T22:10:35.340 INFO:scrape:suites union: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{default}', 'centos_latest', 'clusters/{fixed-2}', 'crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'deploy/ceph', 'objectstore/bluestore', 'thrashers/default', 'thrashosds-health', 'workloads/small-objects}'] 2022-09-08T22:10:35.341 INFO:scrape: