Found 19 jobs Ignoring malparsed backtrace: ceph version 18.0.0-3633-g0ae30b6b (0ae30b6b0a0fdd9fa3bc438c826208438ac27aa2) reef (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7f4ea405ded4] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x1b0) [0x55eaaf7e30e8] Ignoring malparsed backtrace: 2023-04-30T19:12:23.147 INFO:teuthology.orchestra.run.smithi037.stdout: , 2023-04-30T19:12:23.148 INFO:teuthology.orchestra.run.smithi037.stdout:{"epoch":57,"fsid":"d6515b49-3756-4a48-bfb7-6b78510afcb5","created":"2023-04-30T19:10:27.632524+0000","modified":"2023-04-30T19:12:22.780983+0000","last_up_change":"2023-04-30T19:12:11.730472+0000","last_in_change":"2023-04-30T19:10:29.937834+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:10:37.717919+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:10:38.303920+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:10:45.111720+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"47","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":28,"snap_epoch":47,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"63de233e-74c5-4259-8de5-184afa5cb6ea","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":48,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6801","nonce":133312}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6800","nonce":133312}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6804","nonce":133312}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6805","nonce":133312}]},"public_addr":"172.21.15.37:6801/133312","cluster_addr":"172.21.15.37:6800/133312","heartbeat_back_addr":"172.21.15.37:6804/133312","heartbeat_front_addr":"172.21.15.37:6805/133312","state":["exists"]},{"osd":1,"uuid":"669c9d30-fec7-45f7-ad84-7bfc7dd23c9c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6803","nonce":133339}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6802","nonce":133339}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6806","nonce":133339}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6807","nonce":133339}]},"public_addr":"172.21.15.37:6803/133339","cluster_addr":"172.21.15.37:6802/133339","heartbeat_back_addr":"172.21.15.37:6806/133339","heartbeat_front_addr":"172.21.15.37:6807/133339","state":["exists","up"]},{"osd":2,"uuid":"8ca43619-1810-4ed6-8442-14c7d536c8d7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6803","nonce":132949}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6802","nonce":132949}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6807","nonce":132949}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6806","nonce":132949}]},"public_addr":"172.21.15.153:6803/132949","cluster_addr":"172.21.15.153:6802/132949","heartbeat_back_addr":"172.21.15.153:6807/132949","heartbeat_front_addr":"172.21.15.153:6806/132949","state":["exists","up"]},{"osd":3,"uuid":"129e347a-6b4b-496b-a79a-add0f174afcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6805","nonce":132972}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6804","nonce":132972}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6808","nonce":132972}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6809","nonce":132972}]},"public_addr":"172.21.15.153:6805/132972","cluster_addr":"172.21.15.153:6804/132972","heartbeat_back_addr":"172.21.15.153:6808/132972","heartbeat_front_addr":"172.21.15.153:6809/132972","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-04-30T19:12:11.730472+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":5},{"begin":12,"length":1},{"begin":15,"length":1},{"begin":21,"length":1},{"begin":26,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-04-30T19:12:25.692 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~33s Ignoring malparsed backtrace: 2023-04-30T19:13:13.423 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~81s , 2023-04-30T19:13:13.424 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.1 is failed for ~53s , 2023-04-30T19:13:13.424 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~48s Ignoring malparsed backtrace: 2023-04-30T19:13:57.351+0000 7fbe7a783700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-04-30T19:13:57.556 INFO:teuthology.orchestra.run.smithi037.stdout: , 2023-04-30T19:13:57.556 INFO:teuthology.orchestra.run.smithi037.stdout:{"epoch":57,"fsid":"d6515b49-3756-4a48-bfb7-6b78510afcb5","created":"2023-04-30T19:10:27.632524+0000","modified":"2023-04-30T19:12:22.780983+0000","last_up_change":"2023-04-30T19:12:11.730472+0000","last_in_change":"2023-04-30T19:10:29.937834+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:10:37.717919+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:10:38.303920+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:10:45.111720+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"47","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":28,"snap_epoch":47,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"63de233e-74c5-4259-8de5-184afa5cb6ea","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":48,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6801","nonce":133312}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6800","nonce":133312}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6804","nonce":133312}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6805","nonce":133312}]},"public_addr":"172.21.15.37:6801/133312","cluster_addr":"172.21.15.37:6800/133312","heartbeat_back_addr":"172.21.15.37:6804/133312","heartbeat_front_addr":"172.21.15.37:6805/133312","state":["exists"]},{"osd":1,"uuid":"669c9d30-fec7-45f7-ad84-7bfc7dd23c9c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6803","nonce":133339}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6802","nonce":133339}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6806","nonce":133339}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6807","nonce":133339}]},"public_addr":"172.21.15.37:6803/133339","cluster_addr":"172.21.15.37:6802/133339","heartbeat_back_addr":"172.21.15.37:6806/133339","heartbeat_front_addr":"172.21.15.37:6807/133339","state":["exists","up"]},{"osd":2,"uuid":"8ca43619-1810-4ed6-8442-14c7d536c8d7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6803","nonce":132949}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6802","nonce":132949}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6807","nonce":132949}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6806","nonce":132949}]},"public_addr":"172.21.15.153:6803/132949","cluster_addr":"172.21.15.153:6802/132949","heartbeat_back_addr":"172.21.15.153:6807/132949","heartbeat_front_addr":"172.21.15.153:6806/132949","state":["exists","up"]},{"osd":3,"uuid":"129e347a-6b4b-496b-a79a-add0f174afcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6805","nonce":132972}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6804","nonce":132972}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6808","nonce":132972}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6809","nonce":132972}]},"public_addr":"172.21.15.153:6805/132972","cluster_addr":"172.21.15.153:6804/132972","heartbeat_back_addr":"172.21.15.153:6808/132972","heartbeat_front_addr":"172.21.15.153:6809/132972","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-04-30T19:12:11.730472+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":5},{"begin":12,"length":1},{"begin":15,"length":1},{"begin":21,"length":1},{"begin":26,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} Ignoring malparsed backtrace: 2023-04-30T19:14:42.635+0000 7fac43d76700 -1 mon.a@0(leader) e1 get_health_metrics reporting 2 slow ops, oldest is osd_failure(failed timeout osd.1 v2:172.21.15.37:6803/133339 for 29sec e57 v57) , 2023-04-30T19:14:43.585 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~171s , 2023-04-30T19:14:43.585 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.1 is failed for ~143s Ignoring malparsed backtrace: 2023-04-30T19:15:27.643+0000 7fac43d76700 -1 mon.a@0(leader) e1 get_health_metrics reporting 2 slow ops, oldest is osd_failure(failed timeout osd.1 v2:172.21.15.37:6803/133339 for 29sec e57 v57) , 2023-04-30T19:15:28.261 DEBUG:teuthology.orchestra.run.smithi037:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-04-30T19:15:28.406+0000 7ff72fe03700 -1 WARNING: the following dangerous and experimental features are enabled: crimson Ignoring malparsed backtrace: 2023-04-30T19:16:12.460 INFO:teuthology.orchestra.run.smithi037.stdout: , 2023-04-30T19:16:12.460 INFO:teuthology.orchestra.run.smithi037.stdout:{"epoch":57,"fsid":"d6515b49-3756-4a48-bfb7-6b78510afcb5","created":"2023-04-30T19:10:27.632524+0000","modified":"2023-04-30T19:12:22.780983+0000","last_up_change":"2023-04-30T19:12:11.730472+0000","last_in_change":"2023-04-30T19:10:29.937834+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:10:37.717919+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:10:38.303920+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:10:45.111720+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"47","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":28,"snap_epoch":47,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"63de233e-74c5-4259-8de5-184afa5cb6ea","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":48,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6801","nonce":133312}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6800","nonce":133312}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6804","nonce":133312}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6805","nonce":133312}]},"public_addr":"172.21.15.37:6801/133312","cluster_addr":"172.21.15.37:6800/133312","heartbeat_back_addr":"172.21.15.37:6804/133312","heartbeat_front_addr":"172.21.15.37:6805/133312","state":["exists"]},{"osd":1,"uuid":"669c9d30-fec7-45f7-ad84-7bfc7dd23c9c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6803","nonce":133339}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6802","nonce":133339}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6806","nonce":133339}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6807","nonce":133339}]},"public_addr":"172.21.15.37:6803/133339","cluster_addr":"172.21.15.37:6802/133339","heartbeat_back_addr":"172.21.15.37:6806/133339","heartbeat_front_addr":"172.21.15.37:6807/133339","state":["exists","up"]},{"osd":2,"uuid":"8ca43619-1810-4ed6-8442-14c7d536c8d7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6803","nonce":132949}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6802","nonce":132949}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6807","nonce":132949}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6806","nonce":132949}]},"public_addr":"172.21.15.153:6803/132949","cluster_addr":"172.21.15.153:6802/132949","heartbeat_back_addr":"172.21.15.153:6807/132949","heartbeat_front_addr":"172.21.15.153:6806/132949","state":["exists","up"]},{"osd":3,"uuid":"129e347a-6b4b-496b-a79a-add0f174afcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6805","nonce":132972}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6804","nonce":132972}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6808","nonce":132972}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6809","nonce":132972}]},"public_addr":"172.21.15.153:6805/132972","cluster_addr":"172.21.15.153:6804/132972","heartbeat_back_addr":"172.21.15.153:6808/132972","heartbeat_front_addr":"172.21.15.153:6809/132972","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-04-30T19:12:11.730472+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":5},{"begin":12,"length":1},{"begin":15,"length":1},{"begin":21,"length":1},{"begin":26,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-04-30T19:16:12.651+0000 7fac43d76700 -1 mon.a@0(leader) e1 get_health_metrics reporting 2 slow ops, oldest is osd_failure(failed timeout osd.1 v2:172.21.15.37:6803/133339 for 29sec e57 v57) Ignoring malparsed backtrace: 2023-04-30T19:16:56.176 INFO:tasks.daemonwatchdog.daemon_watchdog:BARK! unmounting mounts and killing all daemons , 2023-04-30T19:16:56.307 INFO:teuthology.orchestra.run.smithi037.stdout: , 2023-04-30T19:16:56.307 INFO:teuthology.orchestra.run.smithi037.stdout:{"epoch":57,"fsid":"d6515b49-3756-4a48-bfb7-6b78510afcb5","created":"2023-04-30T19:10:27.632524+0000","modified":"2023-04-30T19:12:22.780983+0000","last_up_change":"2023-04-30T19:12:11.730472+0000","last_in_change":"2023-04-30T19:10:29.937834+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:10:37.717919+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:10:38.303920+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:10:45.111720+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"47","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":28,"snap_epoch":47,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"63de233e-74c5-4259-8de5-184afa5cb6ea","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":48,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6801","nonce":133312}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6800","nonce":133312}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6804","nonce":133312}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6805","nonce":133312}]},"public_addr":"172.21.15.37:6801/133312","cluster_addr":"172.21.15.37:6800/133312","heartbeat_back_addr":"172.21.15.37:6804/133312","heartbeat_front_addr":"172.21.15.37:6805/133312","state":["exists"]},{"osd":1,"uuid":"669c9d30-fec7-45f7-ad84-7bfc7dd23c9c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6803","nonce":133339}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6802","nonce":133339}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6806","nonce":133339}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6807","nonce":133339}]},"public_addr":"172.21.15.37:6803/133339","cluster_addr":"172.21.15.37:6802/133339","heartbeat_back_addr":"172.21.15.37:6806/133339","heartbeat_front_addr":"172.21.15.37:6807/133339","state":["exists","up"]},{"osd":2,"uuid":"8ca43619-1810-4ed6-8442-14c7d536c8d7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6803","nonce":132949}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6802","nonce":132949}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6807","nonce":132949}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6806","nonce":132949}]},"public_addr":"172.21.15.153:6803/132949","cluster_addr":"172.21.15.153:6802/132949","heartbeat_back_addr":"172.21.15.153:6807/132949","heartbeat_front_addr":"172.21.15.153:6806/132949","state":["exists","up"]},{"osd":3,"uuid":"129e347a-6b4b-496b-a79a-add0f174afcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6805","nonce":132972}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6804","nonce":132972}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6808","nonce":132972}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6809","nonce":132972}]},"public_addr":"172.21.15.153:6805/132972","cluster_addr":"172.21.15.153:6804/132972","heartbeat_back_addr":"172.21.15.153:6808/132972","heartbeat_front_addr":"172.21.15.153:6809/132972","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-04-30T19:12:11.730472+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":5},{"begin":12,"length":1},{"begin":15,"length":1},{"begin":21,"length":1},{"begin":26,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} Ignoring malparsed backtrace: run.wait(tests.values()) , File "/home/teuthworker/src/git.ceph.com_teuthology_69caab8ec6f44c61c991ae693e8e7b8cd859a896/teuthology/orchestra/run.py", line 479, in wait , proc.wait() Ignoring malparsed backtrace: 2023-04-30T20:08:59.627 INFO:teuthology.orchestra.run:waiting for 300 , 2023-04-30T20:08:59.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 , 2023-04-30T20:08:59.627 ERROR:teuthology.orchestra.daemon.state:Error while waiting for process to exit Ignoring malparsed backtrace: 2023-04-30T20:08:59.772 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-1 on ubuntu@smithi037.front.sepia.ceph.com , 2023-04-30T20:08:59.773 DEBUG:teuthology.orchestra.run.smithi037:> sync && sudo umount -f /var/lib/ceph/osd/ceph-1 , 2023-04-30T20:08:59.854 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-2 on ubuntu@smithi153.front.sepia.ceph.com Ignoring malparsed backtrace: 2023-04-30T20:09:05.861 INFO:teuthology.task.internal.syslog:Checking logs for errors... , 2023-04-30T20:09:05.862 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi037.front.sepia.ceph.com , 2023-04-30T20:09:05.862 DEBUG:teuthology.orchestra.run.smithi037:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 Ignoring malparsed backtrace: ceph version 18.0.0-3633-g0ae30b6b (0ae30b6b0a0fdd9fa3bc438c826208438ac27aa2) reef (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7f4ea405ded4] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x1b0) [0x55eaaf7e30e8] Ignoring malparsed backtrace: 2023-04-30T19:12:23.147 INFO:teuthology.orchestra.run.smithi037.stdout: , 2023-04-30T19:12:23.148 INFO:teuthology.orchestra.run.smithi037.stdout:{"epoch":57,"fsid":"d6515b49-3756-4a48-bfb7-6b78510afcb5","created":"2023-04-30T19:10:27.632524+0000","modified":"2023-04-30T19:12:22.780983+0000","last_up_change":"2023-04-30T19:12:11.730472+0000","last_in_change":"2023-04-30T19:10:29.937834+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:10:37.717919+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:10:38.303920+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:10:45.111720+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"47","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":28,"snap_epoch":47,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"63de233e-74c5-4259-8de5-184afa5cb6ea","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":48,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6801","nonce":133312}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6800","nonce":133312}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6804","nonce":133312}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6805","nonce":133312}]},"public_addr":"172.21.15.37:6801/133312","cluster_addr":"172.21.15.37:6800/133312","heartbeat_back_addr":"172.21.15.37:6804/133312","heartbeat_front_addr":"172.21.15.37:6805/133312","state":["exists"]},{"osd":1,"uuid":"669c9d30-fec7-45f7-ad84-7bfc7dd23c9c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6803","nonce":133339}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6802","nonce":133339}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6806","nonce":133339}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6807","nonce":133339}]},"public_addr":"172.21.15.37:6803/133339","cluster_addr":"172.21.15.37:6802/133339","heartbeat_back_addr":"172.21.15.37:6806/133339","heartbeat_front_addr":"172.21.15.37:6807/133339","state":["exists","up"]},{"osd":2,"uuid":"8ca43619-1810-4ed6-8442-14c7d536c8d7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6803","nonce":132949}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6802","nonce":132949}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6807","nonce":132949}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6806","nonce":132949}]},"public_addr":"172.21.15.153:6803/132949","cluster_addr":"172.21.15.153:6802/132949","heartbeat_back_addr":"172.21.15.153:6807/132949","heartbeat_front_addr":"172.21.15.153:6806/132949","state":["exists","up"]},{"osd":3,"uuid":"129e347a-6b4b-496b-a79a-add0f174afcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6805","nonce":132972}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6804","nonce":132972}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6808","nonce":132972}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6809","nonce":132972}]},"public_addr":"172.21.15.153:6805/132972","cluster_addr":"172.21.15.153:6804/132972","heartbeat_back_addr":"172.21.15.153:6808/132972","heartbeat_front_addr":"172.21.15.153:6809/132972","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-04-30T19:12:11.730472+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":5},{"begin":12,"length":1},{"begin":15,"length":1},{"begin":21,"length":1},{"begin":26,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-04-30T19:12:25.692 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~33s Ignoring malparsed backtrace: 2023-04-30T19:13:13.423 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~81s , 2023-04-30T19:13:13.424 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.1 is failed for ~53s , 2023-04-30T19:13:13.424 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~48s Ignoring malparsed backtrace: 2023-04-30T19:13:57.351+0000 7fbe7a783700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-04-30T19:13:57.556 INFO:teuthology.orchestra.run.smithi037.stdout: , 2023-04-30T19:13:57.556 INFO:teuthology.orchestra.run.smithi037.stdout:{"epoch":57,"fsid":"d6515b49-3756-4a48-bfb7-6b78510afcb5","created":"2023-04-30T19:10:27.632524+0000","modified":"2023-04-30T19:12:22.780983+0000","last_up_change":"2023-04-30T19:12:11.730472+0000","last_in_change":"2023-04-30T19:10:29.937834+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:10:37.717919+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:10:38.303920+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:10:45.111720+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"47","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":28,"snap_epoch":47,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"63de233e-74c5-4259-8de5-184afa5cb6ea","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":48,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6801","nonce":133312}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6800","nonce":133312}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6804","nonce":133312}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6805","nonce":133312}]},"public_addr":"172.21.15.37:6801/133312","cluster_addr":"172.21.15.37:6800/133312","heartbeat_back_addr":"172.21.15.37:6804/133312","heartbeat_front_addr":"172.21.15.37:6805/133312","state":["exists"]},{"osd":1,"uuid":"669c9d30-fec7-45f7-ad84-7bfc7dd23c9c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6803","nonce":133339}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6802","nonce":133339}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6806","nonce":133339}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6807","nonce":133339}]},"public_addr":"172.21.15.37:6803/133339","cluster_addr":"172.21.15.37:6802/133339","heartbeat_back_addr":"172.21.15.37:6806/133339","heartbeat_front_addr":"172.21.15.37:6807/133339","state":["exists","up"]},{"osd":2,"uuid":"8ca43619-1810-4ed6-8442-14c7d536c8d7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6803","nonce":132949}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6802","nonce":132949}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6807","nonce":132949}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6806","nonce":132949}]},"public_addr":"172.21.15.153:6803/132949","cluster_addr":"172.21.15.153:6802/132949","heartbeat_back_addr":"172.21.15.153:6807/132949","heartbeat_front_addr":"172.21.15.153:6806/132949","state":["exists","up"]},{"osd":3,"uuid":"129e347a-6b4b-496b-a79a-add0f174afcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6805","nonce":132972}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6804","nonce":132972}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6808","nonce":132972}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6809","nonce":132972}]},"public_addr":"172.21.15.153:6805/132972","cluster_addr":"172.21.15.153:6804/132972","heartbeat_back_addr":"172.21.15.153:6808/132972","heartbeat_front_addr":"172.21.15.153:6809/132972","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-04-30T19:12:11.730472+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":5},{"begin":12,"length":1},{"begin":15,"length":1},{"begin":21,"length":1},{"begin":26,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} Ignoring malparsed backtrace: 2023-04-30T19:14:42.635+0000 7fac43d76700 -1 mon.a@0(leader) e1 get_health_metrics reporting 2 slow ops, oldest is osd_failure(failed timeout osd.1 v2:172.21.15.37:6803/133339 for 29sec e57 v57) , 2023-04-30T19:14:43.585 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~171s , 2023-04-30T19:14:43.585 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.1 is failed for ~143s Ignoring malparsed backtrace: 2023-04-30T19:15:27.643+0000 7fac43d76700 -1 mon.a@0(leader) e1 get_health_metrics reporting 2 slow ops, oldest is osd_failure(failed timeout osd.1 v2:172.21.15.37:6803/133339 for 29sec e57 v57) , 2023-04-30T19:15:28.261 DEBUG:teuthology.orchestra.run.smithi037:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-04-30T19:15:28.406+0000 7ff72fe03700 -1 WARNING: the following dangerous and experimental features are enabled: crimson Ignoring malparsed backtrace: 2023-04-30T19:16:12.460 INFO:teuthology.orchestra.run.smithi037.stdout: , 2023-04-30T19:16:12.460 INFO:teuthology.orchestra.run.smithi037.stdout:{"epoch":57,"fsid":"d6515b49-3756-4a48-bfb7-6b78510afcb5","created":"2023-04-30T19:10:27.632524+0000","modified":"2023-04-30T19:12:22.780983+0000","last_up_change":"2023-04-30T19:12:11.730472+0000","last_in_change":"2023-04-30T19:10:29.937834+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:10:37.717919+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:10:38.303920+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:10:45.111720+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"47","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":28,"snap_epoch":47,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"63de233e-74c5-4259-8de5-184afa5cb6ea","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":48,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6801","nonce":133312}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6800","nonce":133312}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6804","nonce":133312}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6805","nonce":133312}]},"public_addr":"172.21.15.37:6801/133312","cluster_addr":"172.21.15.37:6800/133312","heartbeat_back_addr":"172.21.15.37:6804/133312","heartbeat_front_addr":"172.21.15.37:6805/133312","state":["exists"]},{"osd":1,"uuid":"669c9d30-fec7-45f7-ad84-7bfc7dd23c9c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6803","nonce":133339}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6802","nonce":133339}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6806","nonce":133339}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6807","nonce":133339}]},"public_addr":"172.21.15.37:6803/133339","cluster_addr":"172.21.15.37:6802/133339","heartbeat_back_addr":"172.21.15.37:6806/133339","heartbeat_front_addr":"172.21.15.37:6807/133339","state":["exists","up"]},{"osd":2,"uuid":"8ca43619-1810-4ed6-8442-14c7d536c8d7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6803","nonce":132949}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6802","nonce":132949}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6807","nonce":132949}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6806","nonce":132949}]},"public_addr":"172.21.15.153:6803/132949","cluster_addr":"172.21.15.153:6802/132949","heartbeat_back_addr":"172.21.15.153:6807/132949","heartbeat_front_addr":"172.21.15.153:6806/132949","state":["exists","up"]},{"osd":3,"uuid":"129e347a-6b4b-496b-a79a-add0f174afcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6805","nonce":132972}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6804","nonce":132972}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6808","nonce":132972}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6809","nonce":132972}]},"public_addr":"172.21.15.153:6805/132972","cluster_addr":"172.21.15.153:6804/132972","heartbeat_back_addr":"172.21.15.153:6808/132972","heartbeat_front_addr":"172.21.15.153:6809/132972","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-04-30T19:12:11.730472+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":5},{"begin":12,"length":1},{"begin":15,"length":1},{"begin":21,"length":1},{"begin":26,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-04-30T19:16:12.651+0000 7fac43d76700 -1 mon.a@0(leader) e1 get_health_metrics reporting 2 slow ops, oldest is osd_failure(failed timeout osd.1 v2:172.21.15.37:6803/133339 for 29sec e57 v57) Ignoring malparsed backtrace: 2023-04-30T19:16:56.176 INFO:tasks.daemonwatchdog.daemon_watchdog:BARK! unmounting mounts and killing all daemons , 2023-04-30T19:16:56.307 INFO:teuthology.orchestra.run.smithi037.stdout: , 2023-04-30T19:16:56.307 INFO:teuthology.orchestra.run.smithi037.stdout:{"epoch":57,"fsid":"d6515b49-3756-4a48-bfb7-6b78510afcb5","created":"2023-04-30T19:10:27.632524+0000","modified":"2023-04-30T19:12:22.780983+0000","last_up_change":"2023-04-30T19:12:11.730472+0000","last_in_change":"2023-04-30T19:10:29.937834+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:10:37.717919+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:10:38.303920+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:10:45.111720+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"47","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":28,"snap_epoch":47,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"63de233e-74c5-4259-8de5-184afa5cb6ea","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":48,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6801","nonce":133312}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6800","nonce":133312}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6804","nonce":133312}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6805","nonce":133312}]},"public_addr":"172.21.15.37:6801/133312","cluster_addr":"172.21.15.37:6800/133312","heartbeat_back_addr":"172.21.15.37:6804/133312","heartbeat_front_addr":"172.21.15.37:6805/133312","state":["exists"]},{"osd":1,"uuid":"669c9d30-fec7-45f7-ad84-7bfc7dd23c9c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6803","nonce":133339}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6802","nonce":133339}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6806","nonce":133339}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6807","nonce":133339}]},"public_addr":"172.21.15.37:6803/133339","cluster_addr":"172.21.15.37:6802/133339","heartbeat_back_addr":"172.21.15.37:6806/133339","heartbeat_front_addr":"172.21.15.37:6807/133339","state":["exists","up"]},{"osd":2,"uuid":"8ca43619-1810-4ed6-8442-14c7d536c8d7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6803","nonce":132949}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6802","nonce":132949}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6807","nonce":132949}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6806","nonce":132949}]},"public_addr":"172.21.15.153:6803/132949","cluster_addr":"172.21.15.153:6802/132949","heartbeat_back_addr":"172.21.15.153:6807/132949","heartbeat_front_addr":"172.21.15.153:6806/132949","state":["exists","up"]},{"osd":3,"uuid":"129e347a-6b4b-496b-a79a-add0f174afcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6805","nonce":132972}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6804","nonce":132972}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6808","nonce":132972}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6809","nonce":132972}]},"public_addr":"172.21.15.153:6805/132972","cluster_addr":"172.21.15.153:6804/132972","heartbeat_back_addr":"172.21.15.153:6808/132972","heartbeat_front_addr":"172.21.15.153:6809/132972","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-04-30T19:12:11.730472+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":5},{"begin":12,"length":1},{"begin":15,"length":1},{"begin":21,"length":1},{"begin":26,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} Ignoring malparsed backtrace: run.wait(tests.values()) , File "/home/teuthworker/src/git.ceph.com_teuthology_69caab8ec6f44c61c991ae693e8e7b8cd859a896/teuthology/orchestra/run.py", line 479, in wait , proc.wait() Ignoring malparsed backtrace: 2023-04-30T20:08:59.627 INFO:teuthology.orchestra.run:waiting for 300 , 2023-04-30T20:08:59.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 , 2023-04-30T20:08:59.627 ERROR:teuthology.orchestra.daemon.state:Error while waiting for process to exit Ignoring malparsed backtrace: 2023-04-30T20:08:59.772 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-1 on ubuntu@smithi037.front.sepia.ceph.com , 2023-04-30T20:08:59.773 DEBUG:teuthology.orchestra.run.smithi037:> sync && sudo umount -f /var/lib/ceph/osd/ceph-1 , 2023-04-30T20:08:59.854 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-2 on ubuntu@smithi153.front.sepia.ceph.com Ignoring malparsed backtrace: 2023-04-30T20:09:05.861 INFO:teuthology.task.internal.syslog:Checking logs for errors... , 2023-04-30T20:09:05.862 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi037.front.sepia.ceph.com , 2023-04-30T20:09:05.862 DEBUG:teuthology.orchestra.run.smithi037:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 Ignoring malparsed backtrace: ceph version 18.0.0-3633-g0ae30b6b (0ae30b6b0a0fdd9fa3bc438c826208438ac27aa2) reef (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7f4ea405ded4] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x1b0) [0x55eaaf7e30e8] Ignoring malparsed backtrace: 2023-04-30T19:12:23.147 INFO:teuthology.orchestra.run.smithi037.stdout: , 2023-04-30T19:12:23.148 INFO:teuthology.orchestra.run.smithi037.stdout:{"epoch":57,"fsid":"d6515b49-3756-4a48-bfb7-6b78510afcb5","created":"2023-04-30T19:10:27.632524+0000","modified":"2023-04-30T19:12:22.780983+0000","last_up_change":"2023-04-30T19:12:11.730472+0000","last_in_change":"2023-04-30T19:10:29.937834+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:10:37.717919+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:10:38.303920+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:10:45.111720+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"47","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":28,"snap_epoch":47,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"63de233e-74c5-4259-8de5-184afa5cb6ea","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":48,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6801","nonce":133312}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6800","nonce":133312}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6804","nonce":133312}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6805","nonce":133312}]},"public_addr":"172.21.15.37:6801/133312","cluster_addr":"172.21.15.37:6800/133312","heartbeat_back_addr":"172.21.15.37:6804/133312","heartbeat_front_addr":"172.21.15.37:6805/133312","state":["exists"]},{"osd":1,"uuid":"669c9d30-fec7-45f7-ad84-7bfc7dd23c9c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6803","nonce":133339}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6802","nonce":133339}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6806","nonce":133339}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6807","nonce":133339}]},"public_addr":"172.21.15.37:6803/133339","cluster_addr":"172.21.15.37:6802/133339","heartbeat_back_addr":"172.21.15.37:6806/133339","heartbeat_front_addr":"172.21.15.37:6807/133339","state":["exists","up"]},{"osd":2,"uuid":"8ca43619-1810-4ed6-8442-14c7d536c8d7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6803","nonce":132949}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6802","nonce":132949}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6807","nonce":132949}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6806","nonce":132949}]},"public_addr":"172.21.15.153:6803/132949","cluster_addr":"172.21.15.153:6802/132949","heartbeat_back_addr":"172.21.15.153:6807/132949","heartbeat_front_addr":"172.21.15.153:6806/132949","state":["exists","up"]},{"osd":3,"uuid":"129e347a-6b4b-496b-a79a-add0f174afcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6805","nonce":132972}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6804","nonce":132972}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6808","nonce":132972}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6809","nonce":132972}]},"public_addr":"172.21.15.153:6805/132972","cluster_addr":"172.21.15.153:6804/132972","heartbeat_back_addr":"172.21.15.153:6808/132972","heartbeat_front_addr":"172.21.15.153:6809/132972","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-04-30T19:12:11.730472+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":5},{"begin":12,"length":1},{"begin":15,"length":1},{"begin":21,"length":1},{"begin":26,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-04-30T19:12:25.692 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~33s Ignoring malparsed backtrace: 2023-04-30T19:13:13.423 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~81s , 2023-04-30T19:13:13.424 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.1 is failed for ~53s , 2023-04-30T19:13:13.424 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~48s Ignoring malparsed backtrace: 2023-04-30T19:13:57.351+0000 7fbe7a783700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-04-30T19:13:57.556 INFO:teuthology.orchestra.run.smithi037.stdout: , 2023-04-30T19:13:57.556 INFO:teuthology.orchestra.run.smithi037.stdout:{"epoch":57,"fsid":"d6515b49-3756-4a48-bfb7-6b78510afcb5","created":"2023-04-30T19:10:27.632524+0000","modified":"2023-04-30T19:12:22.780983+0000","last_up_change":"2023-04-30T19:12:11.730472+0000","last_in_change":"2023-04-30T19:10:29.937834+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:10:37.717919+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:10:38.303920+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:10:45.111720+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"47","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":28,"snap_epoch":47,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"63de233e-74c5-4259-8de5-184afa5cb6ea","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":48,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6801","nonce":133312}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6800","nonce":133312}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6804","nonce":133312}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6805","nonce":133312}]},"public_addr":"172.21.15.37:6801/133312","cluster_addr":"172.21.15.37:6800/133312","heartbeat_back_addr":"172.21.15.37:6804/133312","heartbeat_front_addr":"172.21.15.37:6805/133312","state":["exists"]},{"osd":1,"uuid":"669c9d30-fec7-45f7-ad84-7bfc7dd23c9c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6803","nonce":133339}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6802","nonce":133339}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6806","nonce":133339}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6807","nonce":133339}]},"public_addr":"172.21.15.37:6803/133339","cluster_addr":"172.21.15.37:6802/133339","heartbeat_back_addr":"172.21.15.37:6806/133339","heartbeat_front_addr":"172.21.15.37:6807/133339","state":["exists","up"]},{"osd":2,"uuid":"8ca43619-1810-4ed6-8442-14c7d536c8d7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6803","nonce":132949}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6802","nonce":132949}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6807","nonce":132949}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6806","nonce":132949}]},"public_addr":"172.21.15.153:6803/132949","cluster_addr":"172.21.15.153:6802/132949","heartbeat_back_addr":"172.21.15.153:6807/132949","heartbeat_front_addr":"172.21.15.153:6806/132949","state":["exists","up"]},{"osd":3,"uuid":"129e347a-6b4b-496b-a79a-add0f174afcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6805","nonce":132972}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6804","nonce":132972}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6808","nonce":132972}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6809","nonce":132972}]},"public_addr":"172.21.15.153:6805/132972","cluster_addr":"172.21.15.153:6804/132972","heartbeat_back_addr":"172.21.15.153:6808/132972","heartbeat_front_addr":"172.21.15.153:6809/132972","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-04-30T19:12:11.730472+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":5},{"begin":12,"length":1},{"begin":15,"length":1},{"begin":21,"length":1},{"begin":26,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} Ignoring malparsed backtrace: 2023-04-30T19:14:42.635+0000 7fac43d76700 -1 mon.a@0(leader) e1 get_health_metrics reporting 2 slow ops, oldest is osd_failure(failed timeout osd.1 v2:172.21.15.37:6803/133339 for 29sec e57 v57) , 2023-04-30T19:14:43.585 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~171s , 2023-04-30T19:14:43.585 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.1 is failed for ~143s Ignoring malparsed backtrace: 2023-04-30T19:15:27.643+0000 7fac43d76700 -1 mon.a@0(leader) e1 get_health_metrics reporting 2 slow ops, oldest is osd_failure(failed timeout osd.1 v2:172.21.15.37:6803/133339 for 29sec e57 v57) , 2023-04-30T19:15:28.261 DEBUG:teuthology.orchestra.run.smithi037:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-04-30T19:15:28.406+0000 7ff72fe03700 -1 WARNING: the following dangerous and experimental features are enabled: crimson Ignoring malparsed backtrace: 2023-04-30T19:16:12.460 INFO:teuthology.orchestra.run.smithi037.stdout: , 2023-04-30T19:16:12.460 INFO:teuthology.orchestra.run.smithi037.stdout:{"epoch":57,"fsid":"d6515b49-3756-4a48-bfb7-6b78510afcb5","created":"2023-04-30T19:10:27.632524+0000","modified":"2023-04-30T19:12:22.780983+0000","last_up_change":"2023-04-30T19:12:11.730472+0000","last_in_change":"2023-04-30T19:10:29.937834+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:10:37.717919+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:10:38.303920+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:10:45.111720+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"47","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":28,"snap_epoch":47,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"63de233e-74c5-4259-8de5-184afa5cb6ea","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":48,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6801","nonce":133312}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6800","nonce":133312}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6804","nonce":133312}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6805","nonce":133312}]},"public_addr":"172.21.15.37:6801/133312","cluster_addr":"172.21.15.37:6800/133312","heartbeat_back_addr":"172.21.15.37:6804/133312","heartbeat_front_addr":"172.21.15.37:6805/133312","state":["exists"]},{"osd":1,"uuid":"669c9d30-fec7-45f7-ad84-7bfc7dd23c9c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6803","nonce":133339}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6802","nonce":133339}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6806","nonce":133339}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6807","nonce":133339}]},"public_addr":"172.21.15.37:6803/133339","cluster_addr":"172.21.15.37:6802/133339","heartbeat_back_addr":"172.21.15.37:6806/133339","heartbeat_front_addr":"172.21.15.37:6807/133339","state":["exists","up"]},{"osd":2,"uuid":"8ca43619-1810-4ed6-8442-14c7d536c8d7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6803","nonce":132949}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6802","nonce":132949}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6807","nonce":132949}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6806","nonce":132949}]},"public_addr":"172.21.15.153:6803/132949","cluster_addr":"172.21.15.153:6802/132949","heartbeat_back_addr":"172.21.15.153:6807/132949","heartbeat_front_addr":"172.21.15.153:6806/132949","state":["exists","up"]},{"osd":3,"uuid":"129e347a-6b4b-496b-a79a-add0f174afcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6805","nonce":132972}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6804","nonce":132972}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6808","nonce":132972}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6809","nonce":132972}]},"public_addr":"172.21.15.153:6805/132972","cluster_addr":"172.21.15.153:6804/132972","heartbeat_back_addr":"172.21.15.153:6808/132972","heartbeat_front_addr":"172.21.15.153:6809/132972","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-04-30T19:12:11.730472+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":5},{"begin":12,"length":1},{"begin":15,"length":1},{"begin":21,"length":1},{"begin":26,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-04-30T19:16:12.651+0000 7fac43d76700 -1 mon.a@0(leader) e1 get_health_metrics reporting 2 slow ops, oldest is osd_failure(failed timeout osd.1 v2:172.21.15.37:6803/133339 for 29sec e57 v57) Ignoring malparsed backtrace: 2023-04-30T19:16:56.176 INFO:tasks.daemonwatchdog.daemon_watchdog:BARK! unmounting mounts and killing all daemons , 2023-04-30T19:16:56.307 INFO:teuthology.orchestra.run.smithi037.stdout: , 2023-04-30T19:16:56.307 INFO:teuthology.orchestra.run.smithi037.stdout:{"epoch":57,"fsid":"d6515b49-3756-4a48-bfb7-6b78510afcb5","created":"2023-04-30T19:10:27.632524+0000","modified":"2023-04-30T19:12:22.780983+0000","last_up_change":"2023-04-30T19:12:11.730472+0000","last_in_change":"2023-04-30T19:10:29.937834+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:10:37.717919+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:10:38.303920+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:10:45.111720+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"47","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":28,"snap_epoch":47,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"63de233e-74c5-4259-8de5-184afa5cb6ea","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":48,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6801","nonce":133312}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6800","nonce":133312}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6804","nonce":133312}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6805","nonce":133312}]},"public_addr":"172.21.15.37:6801/133312","cluster_addr":"172.21.15.37:6800/133312","heartbeat_back_addr":"172.21.15.37:6804/133312","heartbeat_front_addr":"172.21.15.37:6805/133312","state":["exists"]},{"osd":1,"uuid":"669c9d30-fec7-45f7-ad84-7bfc7dd23c9c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6803","nonce":133339}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6802","nonce":133339}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6806","nonce":133339}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6807","nonce":133339}]},"public_addr":"172.21.15.37:6803/133339","cluster_addr":"172.21.15.37:6802/133339","heartbeat_back_addr":"172.21.15.37:6806/133339","heartbeat_front_addr":"172.21.15.37:6807/133339","state":["exists","up"]},{"osd":2,"uuid":"8ca43619-1810-4ed6-8442-14c7d536c8d7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6803","nonce":132949}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6802","nonce":132949}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6807","nonce":132949}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6806","nonce":132949}]},"public_addr":"172.21.15.153:6803/132949","cluster_addr":"172.21.15.153:6802/132949","heartbeat_back_addr":"172.21.15.153:6807/132949","heartbeat_front_addr":"172.21.15.153:6806/132949","state":["exists","up"]},{"osd":3,"uuid":"129e347a-6b4b-496b-a79a-add0f174afcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6805","nonce":132972}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6804","nonce":132972}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6808","nonce":132972}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6809","nonce":132972}]},"public_addr":"172.21.15.153:6805/132972","cluster_addr":"172.21.15.153:6804/132972","heartbeat_back_addr":"172.21.15.153:6808/132972","heartbeat_front_addr":"172.21.15.153:6809/132972","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-04-30T19:12:11.730472+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":5},{"begin":12,"length":1},{"begin":15,"length":1},{"begin":21,"length":1},{"begin":26,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} Ignoring malparsed backtrace: run.wait(tests.values()) , File "/home/teuthworker/src/git.ceph.com_teuthology_69caab8ec6f44c61c991ae693e8e7b8cd859a896/teuthology/orchestra/run.py", line 479, in wait , proc.wait() Ignoring malparsed backtrace: 2023-04-30T20:08:59.627 INFO:teuthology.orchestra.run:waiting for 300 , 2023-04-30T20:08:59.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 , 2023-04-30T20:08:59.627 ERROR:teuthology.orchestra.daemon.state:Error while waiting for process to exit Ignoring malparsed backtrace: 2023-04-30T20:08:59.772 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-1 on ubuntu@smithi037.front.sepia.ceph.com , 2023-04-30T20:08:59.773 DEBUG:teuthology.orchestra.run.smithi037:> sync && sudo umount -f /var/lib/ceph/osd/ceph-1 , 2023-04-30T20:08:59.854 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-2 on ubuntu@smithi153.front.sepia.ceph.com Ignoring malparsed backtrace: 2023-04-30T20:09:05.861 INFO:teuthology.task.internal.syslog:Checking logs for errors... , 2023-04-30T20:09:05.862 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi037.front.sepia.ceph.com , 2023-04-30T20:09:05.862 DEBUG:teuthology.orchestra.run.smithi037:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 Ignoring malparsed backtrace: ceph version 18.0.0-3633-g0ae30b6b (0ae30b6b0a0fdd9fa3bc438c826208438ac27aa2) reef (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7f4ea405ded4] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x1b0) [0x55eaaf7e30e8] Ignoring malparsed backtrace: 2023-04-30T19:12:23.147 INFO:teuthology.orchestra.run.smithi037.stdout: , 2023-04-30T19:12:23.148 INFO:teuthology.orchestra.run.smithi037.stdout:{"epoch":57,"fsid":"d6515b49-3756-4a48-bfb7-6b78510afcb5","created":"2023-04-30T19:10:27.632524+0000","modified":"2023-04-30T19:12:22.780983+0000","last_up_change":"2023-04-30T19:12:11.730472+0000","last_in_change":"2023-04-30T19:10:29.937834+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:10:37.717919+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:10:38.303920+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:10:45.111720+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"47","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":28,"snap_epoch":47,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"63de233e-74c5-4259-8de5-184afa5cb6ea","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":48,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6801","nonce":133312}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6800","nonce":133312}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6804","nonce":133312}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6805","nonce":133312}]},"public_addr":"172.21.15.37:6801/133312","cluster_addr":"172.21.15.37:6800/133312","heartbeat_back_addr":"172.21.15.37:6804/133312","heartbeat_front_addr":"172.21.15.37:6805/133312","state":["exists"]},{"osd":1,"uuid":"669c9d30-fec7-45f7-ad84-7bfc7dd23c9c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6803","nonce":133339}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6802","nonce":133339}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6806","nonce":133339}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6807","nonce":133339}]},"public_addr":"172.21.15.37:6803/133339","cluster_addr":"172.21.15.37:6802/133339","heartbeat_back_addr":"172.21.15.37:6806/133339","heartbeat_front_addr":"172.21.15.37:6807/133339","state":["exists","up"]},{"osd":2,"uuid":"8ca43619-1810-4ed6-8442-14c7d536c8d7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6803","nonce":132949}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6802","nonce":132949}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6807","nonce":132949}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6806","nonce":132949}]},"public_addr":"172.21.15.153:6803/132949","cluster_addr":"172.21.15.153:6802/132949","heartbeat_back_addr":"172.21.15.153:6807/132949","heartbeat_front_addr":"172.21.15.153:6806/132949","state":["exists","up"]},{"osd":3,"uuid":"129e347a-6b4b-496b-a79a-add0f174afcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6805","nonce":132972}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6804","nonce":132972}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6808","nonce":132972}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6809","nonce":132972}]},"public_addr":"172.21.15.153:6805/132972","cluster_addr":"172.21.15.153:6804/132972","heartbeat_back_addr":"172.21.15.153:6808/132972","heartbeat_front_addr":"172.21.15.153:6809/132972","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-04-30T19:12:11.730472+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":5},{"begin":12,"length":1},{"begin":15,"length":1},{"begin":21,"length":1},{"begin":26,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-04-30T19:12:25.692 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~33s Ignoring malparsed backtrace: 2023-04-30T19:13:13.423 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~81s , 2023-04-30T19:13:13.424 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.1 is failed for ~53s , 2023-04-30T19:13:13.424 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~48s Ignoring malparsed backtrace: 2023-04-30T19:13:57.351+0000 7fbe7a783700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-04-30T19:13:57.556 INFO:teuthology.orchestra.run.smithi037.stdout: , 2023-04-30T19:13:57.556 INFO:teuthology.orchestra.run.smithi037.stdout:{"epoch":57,"fsid":"d6515b49-3756-4a48-bfb7-6b78510afcb5","created":"2023-04-30T19:10:27.632524+0000","modified":"2023-04-30T19:12:22.780983+0000","last_up_change":"2023-04-30T19:12:11.730472+0000","last_in_change":"2023-04-30T19:10:29.937834+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:10:37.717919+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:10:38.303920+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:10:45.111720+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"47","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":28,"snap_epoch":47,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"63de233e-74c5-4259-8de5-184afa5cb6ea","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":48,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6801","nonce":133312}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6800","nonce":133312}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6804","nonce":133312}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6805","nonce":133312}]},"public_addr":"172.21.15.37:6801/133312","cluster_addr":"172.21.15.37:6800/133312","heartbeat_back_addr":"172.21.15.37:6804/133312","heartbeat_front_addr":"172.21.15.37:6805/133312","state":["exists"]},{"osd":1,"uuid":"669c9d30-fec7-45f7-ad84-7bfc7dd23c9c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6803","nonce":133339}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6802","nonce":133339}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6806","nonce":133339}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6807","nonce":133339}]},"public_addr":"172.21.15.37:6803/133339","cluster_addr":"172.21.15.37:6802/133339","heartbeat_back_addr":"172.21.15.37:6806/133339","heartbeat_front_addr":"172.21.15.37:6807/133339","state":["exists","up"]},{"osd":2,"uuid":"8ca43619-1810-4ed6-8442-14c7d536c8d7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6803","nonce":132949}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6802","nonce":132949}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6807","nonce":132949}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6806","nonce":132949}]},"public_addr":"172.21.15.153:6803/132949","cluster_addr":"172.21.15.153:6802/132949","heartbeat_back_addr":"172.21.15.153:6807/132949","heartbeat_front_addr":"172.21.15.153:6806/132949","state":["exists","up"]},{"osd":3,"uuid":"129e347a-6b4b-496b-a79a-add0f174afcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6805","nonce":132972}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6804","nonce":132972}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6808","nonce":132972}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6809","nonce":132972}]},"public_addr":"172.21.15.153:6805/132972","cluster_addr":"172.21.15.153:6804/132972","heartbeat_back_addr":"172.21.15.153:6808/132972","heartbeat_front_addr":"172.21.15.153:6809/132972","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-04-30T19:12:11.730472+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":5},{"begin":12,"length":1},{"begin":15,"length":1},{"begin":21,"length":1},{"begin":26,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} Ignoring malparsed backtrace: 2023-04-30T19:14:42.635+0000 7fac43d76700 -1 mon.a@0(leader) e1 get_health_metrics reporting 2 slow ops, oldest is osd_failure(failed timeout osd.1 v2:172.21.15.37:6803/133339 for 29sec e57 v57) , 2023-04-30T19:14:43.585 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~171s , 2023-04-30T19:14:43.585 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.1 is failed for ~143s Ignoring malparsed backtrace: 2023-04-30T19:15:27.643+0000 7fac43d76700 -1 mon.a@0(leader) e1 get_health_metrics reporting 2 slow ops, oldest is osd_failure(failed timeout osd.1 v2:172.21.15.37:6803/133339 for 29sec e57 v57) , 2023-04-30T19:15:28.261 DEBUG:teuthology.orchestra.run.smithi037:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-04-30T19:15:28.406+0000 7ff72fe03700 -1 WARNING: the following dangerous and experimental features are enabled: crimson Ignoring malparsed backtrace: 2023-04-30T19:16:12.460 INFO:teuthology.orchestra.run.smithi037.stdout: , 2023-04-30T19:16:12.460 INFO:teuthology.orchestra.run.smithi037.stdout:{"epoch":57,"fsid":"d6515b49-3756-4a48-bfb7-6b78510afcb5","created":"2023-04-30T19:10:27.632524+0000","modified":"2023-04-30T19:12:22.780983+0000","last_up_change":"2023-04-30T19:12:11.730472+0000","last_in_change":"2023-04-30T19:10:29.937834+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:10:37.717919+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:10:38.303920+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:10:45.111720+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"47","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":28,"snap_epoch":47,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"63de233e-74c5-4259-8de5-184afa5cb6ea","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":48,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6801","nonce":133312}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6800","nonce":133312}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6804","nonce":133312}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6805","nonce":133312}]},"public_addr":"172.21.15.37:6801/133312","cluster_addr":"172.21.15.37:6800/133312","heartbeat_back_addr":"172.21.15.37:6804/133312","heartbeat_front_addr":"172.21.15.37:6805/133312","state":["exists"]},{"osd":1,"uuid":"669c9d30-fec7-45f7-ad84-7bfc7dd23c9c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6803","nonce":133339}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6802","nonce":133339}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6806","nonce":133339}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6807","nonce":133339}]},"public_addr":"172.21.15.37:6803/133339","cluster_addr":"172.21.15.37:6802/133339","heartbeat_back_addr":"172.21.15.37:6806/133339","heartbeat_front_addr":"172.21.15.37:6807/133339","state":["exists","up"]},{"osd":2,"uuid":"8ca43619-1810-4ed6-8442-14c7d536c8d7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6803","nonce":132949}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6802","nonce":132949}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6807","nonce":132949}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6806","nonce":132949}]},"public_addr":"172.21.15.153:6803/132949","cluster_addr":"172.21.15.153:6802/132949","heartbeat_back_addr":"172.21.15.153:6807/132949","heartbeat_front_addr":"172.21.15.153:6806/132949","state":["exists","up"]},{"osd":3,"uuid":"129e347a-6b4b-496b-a79a-add0f174afcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6805","nonce":132972}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6804","nonce":132972}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6808","nonce":132972}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6809","nonce":132972}]},"public_addr":"172.21.15.153:6805/132972","cluster_addr":"172.21.15.153:6804/132972","heartbeat_back_addr":"172.21.15.153:6808/132972","heartbeat_front_addr":"172.21.15.153:6809/132972","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-04-30T19:12:11.730472+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":5},{"begin":12,"length":1},{"begin":15,"length":1},{"begin":21,"length":1},{"begin":26,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-04-30T19:16:12.651+0000 7fac43d76700 -1 mon.a@0(leader) e1 get_health_metrics reporting 2 slow ops, oldest is osd_failure(failed timeout osd.1 v2:172.21.15.37:6803/133339 for 29sec e57 v57) Ignoring malparsed backtrace: 2023-04-30T19:16:56.176 INFO:tasks.daemonwatchdog.daemon_watchdog:BARK! unmounting mounts and killing all daemons , 2023-04-30T19:16:56.307 INFO:teuthology.orchestra.run.smithi037.stdout: , 2023-04-30T19:16:56.307 INFO:teuthology.orchestra.run.smithi037.stdout:{"epoch":57,"fsid":"d6515b49-3756-4a48-bfb7-6b78510afcb5","created":"2023-04-30T19:10:27.632524+0000","modified":"2023-04-30T19:12:22.780983+0000","last_up_change":"2023-04-30T19:12:11.730472+0000","last_in_change":"2023-04-30T19:10:29.937834+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:10:37.717919+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:10:38.303920+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:10:45.111720+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"47","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":28,"snap_epoch":47,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"63de233e-74c5-4259-8de5-184afa5cb6ea","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":48,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6801","nonce":133312}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6800","nonce":133312}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6804","nonce":133312}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6805","nonce":133312}]},"public_addr":"172.21.15.37:6801/133312","cluster_addr":"172.21.15.37:6800/133312","heartbeat_back_addr":"172.21.15.37:6804/133312","heartbeat_front_addr":"172.21.15.37:6805/133312","state":["exists"]},{"osd":1,"uuid":"669c9d30-fec7-45f7-ad84-7bfc7dd23c9c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6803","nonce":133339}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6802","nonce":133339}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6806","nonce":133339}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.37:6807","nonce":133339}]},"public_addr":"172.21.15.37:6803/133339","cluster_addr":"172.21.15.37:6802/133339","heartbeat_back_addr":"172.21.15.37:6806/133339","heartbeat_front_addr":"172.21.15.37:6807/133339","state":["exists","up"]},{"osd":2,"uuid":"8ca43619-1810-4ed6-8442-14c7d536c8d7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6803","nonce":132949}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6802","nonce":132949}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6807","nonce":132949}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6806","nonce":132949}]},"public_addr":"172.21.15.153:6803/132949","cluster_addr":"172.21.15.153:6802/132949","heartbeat_back_addr":"172.21.15.153:6807/132949","heartbeat_front_addr":"172.21.15.153:6806/132949","state":["exists","up"]},{"osd":3,"uuid":"129e347a-6b4b-496b-a79a-add0f174afcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6805","nonce":132972}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6804","nonce":132972}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6808","nonce":132972}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.153:6809","nonce":132972}]},"public_addr":"172.21.15.153:6805/132972","cluster_addr":"172.21.15.153:6804/132972","heartbeat_back_addr":"172.21.15.153:6808/132972","heartbeat_front_addr":"172.21.15.153:6809/132972","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-04-30T19:12:11.730472+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":5},{"begin":12,"length":1},{"begin":15,"length":1},{"begin":21,"length":1},{"begin":26,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} Ignoring malparsed backtrace: run.wait(tests.values()) , File "/home/teuthworker/src/git.ceph.com_teuthology_69caab8ec6f44c61c991ae693e8e7b8cd859a896/teuthology/orchestra/run.py", line 479, in wait , proc.wait() Ignoring malparsed backtrace: 2023-04-30T20:08:59.627 INFO:teuthology.orchestra.run:waiting for 300 , 2023-04-30T20:08:59.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 , 2023-04-30T20:08:59.627 ERROR:teuthology.orchestra.daemon.state:Error while waiting for process to exit Ignoring malparsed backtrace: 2023-04-30T20:08:59.772 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-1 on ubuntu@smithi037.front.sepia.ceph.com , 2023-04-30T20:08:59.773 DEBUG:teuthology.orchestra.run.smithi037:> sync && sudo umount -f /var/lib/ceph/osd/ceph-1 , 2023-04-30T20:08:59.854 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-2 on ubuntu@smithi153.front.sepia.ceph.com Ignoring malparsed backtrace: 2023-04-30T20:09:05.861 INFO:teuthology.task.internal.syslog:Checking logs for errors... , 2023-04-30T20:09:05.862 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi037.front.sepia.ceph.com , 2023-04-30T20:09:05.862 DEBUG:teuthology.orchestra.run.smithi037:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 Ignoring malparsed backtrace: ceph version 18.0.0-3633-g0ae30b6b (0ae30b6b0a0fdd9fa3bc438c826208438ac27aa2) reef (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7fbb1a71bed4] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x1b0) [0x558a06a1d0e8] Ignoring malparsed backtrace: 2023-04-30T19:07:11.683+0000 7f1468f55700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-04-30T19:07:11.692+0000 7f1468f55700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , set osd.3 primary-affinity to 1 (8655362) Ignoring malparsed backtrace: 2023-04-30T19:07:53.367 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":142,"fsid":"748dd0c6-7174-4734-8b97-28004cdc5078","created":"2023-04-30T19:04:13.497850+0000","modified":"2023-04-30T19:07:12.533499+0000","last_up_change":"2023-04-30T19:06:56.619249+0000","last_in_change":"2023-04-30T19:04:15.788663+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:04:23.469100+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.75,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:04:24.177609+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:04:31.063009+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"134","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":116,"snap_epoch":134,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"d50c1949-1aed-45d6-b9f6-b8428fee01d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6801","nonce":133213}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":133213}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6805","nonce":133213}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":133213}]},"public_addr":"172.21.15.33:6801/133213","cluster_addr":"172.21.15.33:6800/133213","heartbeat_back_addr":"172.21.15.33:6805/133213","heartbeat_front_addr":"172.21.15.33:6804/133213","state":["exists","up"]},{"osd":1,"uuid":"1ae75fa1-248f-4f54-8b4a-4fa216c679a3","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":131,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6803","nonce":133240}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":133240}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":133240}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6807","nonce":133240}]},"public_addr":"172.21.15.33:6803/133240","cluster_addr":"172.21.15.33:6802/133240","heartbeat_back_addr":"172.21.15.33:6806/133240","heartbeat_front_addr":"172.21.15.33:6807/133240","state":["exists"]},{"osd":2,"uuid":"13c070d4-a782-403e-9494-13d35a1fd8ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6803","nonce":132825}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":132825}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":132825}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6807","nonce":132825}]},"public_addr":"172.21.15.83:6803/132825","cluster_addr":"172.21.15.83:6802/132825","heartbeat_back_addr":"172.21.15.83:6806/132825","heartbeat_front_addr":"172.21.15.83:6807/132825","state":["exists","up"]},{"osd":3,"uuid":"69ba14d8-63f6-4de7-83e9-9504e8192b8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6805","nonce":132843}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":132843}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":132843}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6809","nonce":132843}]},"public_addr":"172.21.15.83:6805/132843","cluster_addr":"172.21.15.83:6804/132843","heartbeat_back_addr":"172.21.15.83:6808/132843","heartbeat_front_addr":"172.21.15.83:6809/132843","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-04-30T19:06:56.619249+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":6,"length":2},{"begin":10,"length":1},{"begin":12,"length":3},{"begin":16,"length":1},{"begin":18,"length":1},{"begin":21,"length":1},{"begin":24,"length":2},{"begin":28,"length":1},{"begin":30,"length":3},{"begin":34,"length":1},{"begin":38,"length":3},{"begin":43,"length":1},{"begin":45,"length":2},{"begin":48,"length":2},{"begin":52,"length":2},{"begin":55,"length":2},{"begin":58,"length":1},{"begin":60,"length":3},{"begin":64,"length":1},{"begin":66,"length":1},{"begin":68,"length":4},{"begin":75,"length":2},{"begin":78,"length":1},{"begin":80,"length":3},{"begin":85,"length":1},{"begin":90,"length":1},{"begin":97,"length":1},{"begin":99,"length":1},{"begin":106,"length":1},{"begin":108,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-04-30T19:07:54.648 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~47s , 2023-04-30T19:07:54.648 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.1 is failed for ~80s Ignoring malparsed backtrace: 2023-04-30T19:08:37.213 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":142,"fsid":"748dd0c6-7174-4734-8b97-28004cdc5078","created":"2023-04-30T19:04:13.497850+0000","modified":"2023-04-30T19:07:12.533499+0000","last_up_change":"2023-04-30T19:06:56.619249+0000","last_in_change":"2023-04-30T19:04:15.788663+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:04:23.469100+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.75,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:04:24.177609+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:04:31.063009+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"134","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":116,"snap_epoch":134,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"d50c1949-1aed-45d6-b9f6-b8428fee01d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6801","nonce":133213}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":133213}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6805","nonce":133213}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":133213}]},"public_addr":"172.21.15.33:6801/133213","cluster_addr":"172.21.15.33:6800/133213","heartbeat_back_addr":"172.21.15.33:6805/133213","heartbeat_front_addr":"172.21.15.33:6804/133213","state":["exists","up"]},{"osd":1,"uuid":"1ae75fa1-248f-4f54-8b4a-4fa216c679a3","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":131,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6803","nonce":133240}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":133240}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":133240}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6807","nonce":133240}]},"public_addr":"172.21.15.33:6803/133240","cluster_addr":"172.21.15.33:6802/133240","heartbeat_back_addr":"172.21.15.33:6806/133240","heartbeat_front_addr":"172.21.15.33:6807/133240","state":["exists"]},{"osd":2,"uuid":"13c070d4-a782-403e-9494-13d35a1fd8ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6803","nonce":132825}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":132825}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":132825}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6807","nonce":132825}]},"public_addr":"172.21.15.83:6803/132825","cluster_addr":"172.21.15.83:6802/132825","heartbeat_back_addr":"172.21.15.83:6806/132825","heartbeat_front_addr":"172.21.15.83:6807/132825","state":["exists","up"]},{"osd":3,"uuid":"69ba14d8-63f6-4de7-83e9-9504e8192b8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6805","nonce":132843}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":132843}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":132843}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6809","nonce":132843}]},"public_addr":"172.21.15.83:6805/132843","cluster_addr":"172.21.15.83:6804/132843","heartbeat_back_addr":"172.21.15.83:6808/132843","heartbeat_front_addr":"172.21.15.83:6809/132843","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-04-30T19:06:56.619249+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":6,"length":2},{"begin":10,"length":1},{"begin":12,"length":3},{"begin":16,"length":1},{"begin":18,"length":1},{"begin":21,"length":1},{"begin":24,"length":2},{"begin":28,"length":1},{"begin":30,"length":3},{"begin":34,"length":1},{"begin":38,"length":3},{"begin":43,"length":1},{"begin":45,"length":2},{"begin":48,"length":2},{"begin":52,"length":2},{"begin":55,"length":2},{"begin":58,"length":1},{"begin":60,"length":3},{"begin":64,"length":1},{"begin":66,"length":1},{"begin":68,"length":4},{"begin":75,"length":2},{"begin":78,"length":1},{"begin":80,"length":3},{"begin":85,"length":1},{"begin":90,"length":1},{"begin":97,"length":1},{"begin":99,"length":1},{"begin":106,"length":1},{"begin":108,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-04-30T19:08:40.228 DEBUG:teuthology.orchestra.run.smithi033:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-04-30T19:08:40.374+0000 7f39e28d1700 -1 WARNING: the following dangerous and experimental features are enabled: crimson Ignoring malparsed backtrace: 2023-04-30T19:09:23.095 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~130s , 2023-04-30T19:09:24.075 DEBUG:teuthology.orchestra.run.smithi033:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-04-30T19:09:24.219+0000 7f655196c700 -1 WARNING: the following dangerous and experimental features are enabled: crimson Ignoring malparsed backtrace: 2023-04-30T19:10:08.280 INFO:teuthology.orchestra.run.smithi033.stdout: , 2023-04-30T19:10:08.280 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":142,"fsid":"748dd0c6-7174-4734-8b97-28004cdc5078","created":"2023-04-30T19:04:13.497850+0000","modified":"2023-04-30T19:07:12.533499+0000","last_up_change":"2023-04-30T19:06:56.619249+0000","last_in_change":"2023-04-30T19:04:15.788663+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:04:23.469100+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.75,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:04:24.177609+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:04:31.063009+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"134","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":116,"snap_epoch":134,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"d50c1949-1aed-45d6-b9f6-b8428fee01d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6801","nonce":133213}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":133213}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6805","nonce":133213}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":133213}]},"public_addr":"172.21.15.33:6801/133213","cluster_addr":"172.21.15.33:6800/133213","heartbeat_back_addr":"172.21.15.33:6805/133213","heartbeat_front_addr":"172.21.15.33:6804/133213","state":["exists","up"]},{"osd":1,"uuid":"1ae75fa1-248f-4f54-8b4a-4fa216c679a3","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":131,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6803","nonce":133240}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":133240}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":133240}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6807","nonce":133240}]},"public_addr":"172.21.15.33:6803/133240","cluster_addr":"172.21.15.33:6802/133240","heartbeat_back_addr":"172.21.15.33:6806/133240","heartbeat_front_addr":"172.21.15.33:6807/133240","state":["exists"]},{"osd":2,"uuid":"13c070d4-a782-403e-9494-13d35a1fd8ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6803","nonce":132825}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":132825}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":132825}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6807","nonce":132825}]},"public_addr":"172.21.15.83:6803/132825","cluster_addr":"172.21.15.83:6802/132825","heartbeat_back_addr":"172.21.15.83:6806/132825","heartbeat_front_addr":"172.21.15.83:6807/132825","state":["exists","up"]},{"osd":3,"uuid":"69ba14d8-63f6-4de7-83e9-9504e8192b8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6805","nonce":132843}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":132843}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":132843}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6809","nonce":132843}]},"public_addr":"172.21.15.83:6805/132843","cluster_addr":"172.21.15.83:6804/132843","heartbeat_back_addr":"172.21.15.83:6808/132843","heartbeat_front_addr":"172.21.15.83:6809/132843","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-04-30T19:06:56.619249+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":6,"length":2},{"begin":10,"length":1},{"begin":12,"length":3},{"begin":16,"length":1},{"begin":18,"length":1},{"begin":21,"length":1},{"begin":24,"length":2},{"begin":28,"length":1},{"begin":30,"length":3},{"begin":34,"length":1},{"begin":38,"length":3},{"begin":43,"length":1},{"begin":45,"length":2},{"begin":48,"length":2},{"begin":52,"length":2},{"begin":55,"length":2},{"begin":58,"length":1},{"begin":60,"length":3},{"begin":64,"length":1},{"begin":66,"length":1},{"begin":68,"length":4},{"begin":75,"length":2},{"begin":78,"length":1},{"begin":80,"length":3},{"begin":85,"length":1},{"begin":90,"length":1},{"begin":97,"length":1},{"begin":99,"length":1},{"begin":106,"length":1},{"begin":108,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-04-30T19:10:09.933 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~182s Ignoring malparsed backtrace: 2023-04-30T19:10:52.120 INFO:teuthology.orchestra.run.smithi033.stdout: , 2023-04-30T19:10:52.120 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":142,"fsid":"748dd0c6-7174-4734-8b97-28004cdc5078","created":"2023-04-30T19:04:13.497850+0000","modified":"2023-04-30T19:07:12.533499+0000","last_up_change":"2023-04-30T19:06:56.619249+0000","last_in_change":"2023-04-30T19:04:15.788663+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:04:23.469100+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.75,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:04:24.177609+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:04:31.063009+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"134","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":116,"snap_epoch":134,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"d50c1949-1aed-45d6-b9f6-b8428fee01d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6801","nonce":133213}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":133213}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6805","nonce":133213}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":133213}]},"public_addr":"172.21.15.33:6801/133213","cluster_addr":"172.21.15.33:6800/133213","heartbeat_back_addr":"172.21.15.33:6805/133213","heartbeat_front_addr":"172.21.15.33:6804/133213","state":["exists","up"]},{"osd":1,"uuid":"1ae75fa1-248f-4f54-8b4a-4fa216c679a3","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":131,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6803","nonce":133240}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":133240}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":133240}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6807","nonce":133240}]},"public_addr":"172.21.15.33:6803/133240","cluster_addr":"172.21.15.33:6802/133240","heartbeat_back_addr":"172.21.15.33:6806/133240","heartbeat_front_addr":"172.21.15.33:6807/133240","state":["exists"]},{"osd":2,"uuid":"13c070d4-a782-403e-9494-13d35a1fd8ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6803","nonce":132825}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":132825}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":132825}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6807","nonce":132825}]},"public_addr":"172.21.15.83:6803/132825","cluster_addr":"172.21.15.83:6802/132825","heartbeat_back_addr":"172.21.15.83:6806/132825","heartbeat_front_addr":"172.21.15.83:6807/132825","state":["exists","up"]},{"osd":3,"uuid":"69ba14d8-63f6-4de7-83e9-9504e8192b8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6805","nonce":132843}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":132843}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":132843}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6809","nonce":132843}]},"public_addr":"172.21.15.83:6805/132843","cluster_addr":"172.21.15.83:6804/132843","heartbeat_back_addr":"172.21.15.83:6808/132843","heartbeat_front_addr":"172.21.15.83:6809/132843","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-04-30T19:06:56.619249+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":6,"length":2},{"begin":10,"length":1},{"begin":12,"length":3},{"begin":16,"length":1},{"begin":18,"length":1},{"begin":21,"length":1},{"begin":24,"length":2},{"begin":28,"length":1},{"begin":30,"length":3},{"begin":34,"length":1},{"begin":38,"length":3},{"begin":43,"length":1},{"begin":45,"length":2},{"begin":48,"length":2},{"begin":52,"length":2},{"begin":55,"length":2},{"begin":58,"length":1},{"begin":60,"length":3},{"begin":64,"length":1},{"begin":66,"length":1},{"begin":68,"length":4},{"begin":75,"length":2},{"begin":78,"length":1},{"begin":80,"length":3},{"begin":85,"length":1},{"begin":90,"length":1},{"begin":97,"length":1},{"begin":99,"length":1},{"begin":106,"length":1},{"begin":108,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-04-30T19:10:55.135 DEBUG:teuthology.orchestra.run.smithi033:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json Ignoring malparsed backtrace: 2023-04-30T19:11:38.383 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.2 is failed for ~265s , 2023-04-30T19:11:38.383 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~265s , 2023-04-30T19:11:38.383 INFO:tasks.daemonwatchdog.daemon_watchdog:BARK! unmounting mounts and killing all daemons Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_teuthology_69caab8ec6f44c61c991ae693e8e7b8cd859a896/teuthology/orchestra/run.py", line 479, in wait , proc.wait() , File "/home/teuthworker/src/git.ceph.com_teuthology_69caab8ec6f44c61c991ae693e8e7b8cd859a896/teuthology/orchestra/run.py", line 161, in wait Ignoring malparsed backtrace: 2023-04-30T20:03:39.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 , 2023-04-30T20:03:39.220 ERROR:teuthology.orchestra.daemon.state:Error while waiting for process to exit , Traceback (most recent call last): Ignoring malparsed backtrace: 2023-04-30T20:03:39.398 DEBUG:teuthology.orchestra.run.smithi033:> sync && sudo umount -f /var/lib/ceph/osd/ceph-1 , 2023-04-30T20:03:39.461 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-2 on ubuntu@smithi083.front.sepia.ceph.com , 2023-04-30T20:03:39.462 DEBUG:teuthology.orchestra.run.smithi083:> sync && sudo umount -f /var/lib/ceph/osd/ceph-2 Ignoring malparsed backtrace: 2023-04-30T20:03:43.072 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi033.front.sepia.ceph.com , 2023-04-30T20:03:43.073 DEBUG:teuthology.orchestra.run.smithi033:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 , 2023-04-30T20:03:43.097 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi083.front.sepia.ceph.com Ignoring malparsed backtrace: ceph version 18.0.0-3633-g0ae30b6b (0ae30b6b0a0fdd9fa3bc438c826208438ac27aa2) reef (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7fbb1a71bed4] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x1b0) [0x558a06a1d0e8] Ignoring malparsed backtrace: 2023-04-30T19:07:11.683+0000 7f1468f55700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-04-30T19:07:11.692+0000 7f1468f55700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , set osd.3 primary-affinity to 1 (8655362) Ignoring malparsed backtrace: 2023-04-30T19:07:53.367 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":142,"fsid":"748dd0c6-7174-4734-8b97-28004cdc5078","created":"2023-04-30T19:04:13.497850+0000","modified":"2023-04-30T19:07:12.533499+0000","last_up_change":"2023-04-30T19:06:56.619249+0000","last_in_change":"2023-04-30T19:04:15.788663+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:04:23.469100+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.75,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:04:24.177609+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:04:31.063009+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"134","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":116,"snap_epoch":134,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"d50c1949-1aed-45d6-b9f6-b8428fee01d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6801","nonce":133213}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":133213}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6805","nonce":133213}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":133213}]},"public_addr":"172.21.15.33:6801/133213","cluster_addr":"172.21.15.33:6800/133213","heartbeat_back_addr":"172.21.15.33:6805/133213","heartbeat_front_addr":"172.21.15.33:6804/133213","state":["exists","up"]},{"osd":1,"uuid":"1ae75fa1-248f-4f54-8b4a-4fa216c679a3","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":131,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6803","nonce":133240}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":133240}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":133240}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6807","nonce":133240}]},"public_addr":"172.21.15.33:6803/133240","cluster_addr":"172.21.15.33:6802/133240","heartbeat_back_addr":"172.21.15.33:6806/133240","heartbeat_front_addr":"172.21.15.33:6807/133240","state":["exists"]},{"osd":2,"uuid":"13c070d4-a782-403e-9494-13d35a1fd8ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6803","nonce":132825}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":132825}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":132825}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6807","nonce":132825}]},"public_addr":"172.21.15.83:6803/132825","cluster_addr":"172.21.15.83:6802/132825","heartbeat_back_addr":"172.21.15.83:6806/132825","heartbeat_front_addr":"172.21.15.83:6807/132825","state":["exists","up"]},{"osd":3,"uuid":"69ba14d8-63f6-4de7-83e9-9504e8192b8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6805","nonce":132843}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":132843}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":132843}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6809","nonce":132843}]},"public_addr":"172.21.15.83:6805/132843","cluster_addr":"172.21.15.83:6804/132843","heartbeat_back_addr":"172.21.15.83:6808/132843","heartbeat_front_addr":"172.21.15.83:6809/132843","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-04-30T19:06:56.619249+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":6,"length":2},{"begin":10,"length":1},{"begin":12,"length":3},{"begin":16,"length":1},{"begin":18,"length":1},{"begin":21,"length":1},{"begin":24,"length":2},{"begin":28,"length":1},{"begin":30,"length":3},{"begin":34,"length":1},{"begin":38,"length":3},{"begin":43,"length":1},{"begin":45,"length":2},{"begin":48,"length":2},{"begin":52,"length":2},{"begin":55,"length":2},{"begin":58,"length":1},{"begin":60,"length":3},{"begin":64,"length":1},{"begin":66,"length":1},{"begin":68,"length":4},{"begin":75,"length":2},{"begin":78,"length":1},{"begin":80,"length":3},{"begin":85,"length":1},{"begin":90,"length":1},{"begin":97,"length":1},{"begin":99,"length":1},{"begin":106,"length":1},{"begin":108,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-04-30T19:07:54.648 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~47s , 2023-04-30T19:07:54.648 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.1 is failed for ~80s Ignoring malparsed backtrace: 2023-04-30T19:08:37.213 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":142,"fsid":"748dd0c6-7174-4734-8b97-28004cdc5078","created":"2023-04-30T19:04:13.497850+0000","modified":"2023-04-30T19:07:12.533499+0000","last_up_change":"2023-04-30T19:06:56.619249+0000","last_in_change":"2023-04-30T19:04:15.788663+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:04:23.469100+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.75,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:04:24.177609+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:04:31.063009+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"134","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":116,"snap_epoch":134,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"d50c1949-1aed-45d6-b9f6-b8428fee01d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6801","nonce":133213}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":133213}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6805","nonce":133213}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":133213}]},"public_addr":"172.21.15.33:6801/133213","cluster_addr":"172.21.15.33:6800/133213","heartbeat_back_addr":"172.21.15.33:6805/133213","heartbeat_front_addr":"172.21.15.33:6804/133213","state":["exists","up"]},{"osd":1,"uuid":"1ae75fa1-248f-4f54-8b4a-4fa216c679a3","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":131,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6803","nonce":133240}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":133240}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":133240}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6807","nonce":133240}]},"public_addr":"172.21.15.33:6803/133240","cluster_addr":"172.21.15.33:6802/133240","heartbeat_back_addr":"172.21.15.33:6806/133240","heartbeat_front_addr":"172.21.15.33:6807/133240","state":["exists"]},{"osd":2,"uuid":"13c070d4-a782-403e-9494-13d35a1fd8ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6803","nonce":132825}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":132825}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":132825}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6807","nonce":132825}]},"public_addr":"172.21.15.83:6803/132825","cluster_addr":"172.21.15.83:6802/132825","heartbeat_back_addr":"172.21.15.83:6806/132825","heartbeat_front_addr":"172.21.15.83:6807/132825","state":["exists","up"]},{"osd":3,"uuid":"69ba14d8-63f6-4de7-83e9-9504e8192b8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6805","nonce":132843}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":132843}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":132843}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6809","nonce":132843}]},"public_addr":"172.21.15.83:6805/132843","cluster_addr":"172.21.15.83:6804/132843","heartbeat_back_addr":"172.21.15.83:6808/132843","heartbeat_front_addr":"172.21.15.83:6809/132843","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-04-30T19:06:56.619249+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":6,"length":2},{"begin":10,"length":1},{"begin":12,"length":3},{"begin":16,"length":1},{"begin":18,"length":1},{"begin":21,"length":1},{"begin":24,"length":2},{"begin":28,"length":1},{"begin":30,"length":3},{"begin":34,"length":1},{"begin":38,"length":3},{"begin":43,"length":1},{"begin":45,"length":2},{"begin":48,"length":2},{"begin":52,"length":2},{"begin":55,"length":2},{"begin":58,"length":1},{"begin":60,"length":3},{"begin":64,"length":1},{"begin":66,"length":1},{"begin":68,"length":4},{"begin":75,"length":2},{"begin":78,"length":1},{"begin":80,"length":3},{"begin":85,"length":1},{"begin":90,"length":1},{"begin":97,"length":1},{"begin":99,"length":1},{"begin":106,"length":1},{"begin":108,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-04-30T19:08:40.228 DEBUG:teuthology.orchestra.run.smithi033:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-04-30T19:08:40.374+0000 7f39e28d1700 -1 WARNING: the following dangerous and experimental features are enabled: crimson Ignoring malparsed backtrace: 2023-04-30T19:09:23.095 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~130s , 2023-04-30T19:09:24.075 DEBUG:teuthology.orchestra.run.smithi033:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-04-30T19:09:24.219+0000 7f655196c700 -1 WARNING: the following dangerous and experimental features are enabled: crimson Ignoring malparsed backtrace: 2023-04-30T19:10:08.280 INFO:teuthology.orchestra.run.smithi033.stdout: , 2023-04-30T19:10:08.280 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":142,"fsid":"748dd0c6-7174-4734-8b97-28004cdc5078","created":"2023-04-30T19:04:13.497850+0000","modified":"2023-04-30T19:07:12.533499+0000","last_up_change":"2023-04-30T19:06:56.619249+0000","last_in_change":"2023-04-30T19:04:15.788663+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:04:23.469100+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.75,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:04:24.177609+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:04:31.063009+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"134","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":116,"snap_epoch":134,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"d50c1949-1aed-45d6-b9f6-b8428fee01d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6801","nonce":133213}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":133213}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6805","nonce":133213}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":133213}]},"public_addr":"172.21.15.33:6801/133213","cluster_addr":"172.21.15.33:6800/133213","heartbeat_back_addr":"172.21.15.33:6805/133213","heartbeat_front_addr":"172.21.15.33:6804/133213","state":["exists","up"]},{"osd":1,"uuid":"1ae75fa1-248f-4f54-8b4a-4fa216c679a3","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":131,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6803","nonce":133240}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":133240}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":133240}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6807","nonce":133240}]},"public_addr":"172.21.15.33:6803/133240","cluster_addr":"172.21.15.33:6802/133240","heartbeat_back_addr":"172.21.15.33:6806/133240","heartbeat_front_addr":"172.21.15.33:6807/133240","state":["exists"]},{"osd":2,"uuid":"13c070d4-a782-403e-9494-13d35a1fd8ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6803","nonce":132825}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":132825}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":132825}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6807","nonce":132825}]},"public_addr":"172.21.15.83:6803/132825","cluster_addr":"172.21.15.83:6802/132825","heartbeat_back_addr":"172.21.15.83:6806/132825","heartbeat_front_addr":"172.21.15.83:6807/132825","state":["exists","up"]},{"osd":3,"uuid":"69ba14d8-63f6-4de7-83e9-9504e8192b8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6805","nonce":132843}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":132843}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":132843}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6809","nonce":132843}]},"public_addr":"172.21.15.83:6805/132843","cluster_addr":"172.21.15.83:6804/132843","heartbeat_back_addr":"172.21.15.83:6808/132843","heartbeat_front_addr":"172.21.15.83:6809/132843","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-04-30T19:06:56.619249+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":6,"length":2},{"begin":10,"length":1},{"begin":12,"length":3},{"begin":16,"length":1},{"begin":18,"length":1},{"begin":21,"length":1},{"begin":24,"length":2},{"begin":28,"length":1},{"begin":30,"length":3},{"begin":34,"length":1},{"begin":38,"length":3},{"begin":43,"length":1},{"begin":45,"length":2},{"begin":48,"length":2},{"begin":52,"length":2},{"begin":55,"length":2},{"begin":58,"length":1},{"begin":60,"length":3},{"begin":64,"length":1},{"begin":66,"length":1},{"begin":68,"length":4},{"begin":75,"length":2},{"begin":78,"length":1},{"begin":80,"length":3},{"begin":85,"length":1},{"begin":90,"length":1},{"begin":97,"length":1},{"begin":99,"length":1},{"begin":106,"length":1},{"begin":108,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-04-30T19:10:09.933 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~182s Ignoring malparsed backtrace: 2023-04-30T19:10:52.120 INFO:teuthology.orchestra.run.smithi033.stdout: , 2023-04-30T19:10:52.120 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":142,"fsid":"748dd0c6-7174-4734-8b97-28004cdc5078","created":"2023-04-30T19:04:13.497850+0000","modified":"2023-04-30T19:07:12.533499+0000","last_up_change":"2023-04-30T19:06:56.619249+0000","last_in_change":"2023-04-30T19:04:15.788663+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:04:23.469100+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.75,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:04:24.177609+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:04:31.063009+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"134","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":116,"snap_epoch":134,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"d50c1949-1aed-45d6-b9f6-b8428fee01d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6801","nonce":133213}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":133213}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6805","nonce":133213}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":133213}]},"public_addr":"172.21.15.33:6801/133213","cluster_addr":"172.21.15.33:6800/133213","heartbeat_back_addr":"172.21.15.33:6805/133213","heartbeat_front_addr":"172.21.15.33:6804/133213","state":["exists","up"]},{"osd":1,"uuid":"1ae75fa1-248f-4f54-8b4a-4fa216c679a3","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":131,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6803","nonce":133240}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":133240}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":133240}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6807","nonce":133240}]},"public_addr":"172.21.15.33:6803/133240","cluster_addr":"172.21.15.33:6802/133240","heartbeat_back_addr":"172.21.15.33:6806/133240","heartbeat_front_addr":"172.21.15.33:6807/133240","state":["exists"]},{"osd":2,"uuid":"13c070d4-a782-403e-9494-13d35a1fd8ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6803","nonce":132825}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":132825}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":132825}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6807","nonce":132825}]},"public_addr":"172.21.15.83:6803/132825","cluster_addr":"172.21.15.83:6802/132825","heartbeat_back_addr":"172.21.15.83:6806/132825","heartbeat_front_addr":"172.21.15.83:6807/132825","state":["exists","up"]},{"osd":3,"uuid":"69ba14d8-63f6-4de7-83e9-9504e8192b8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6805","nonce":132843}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":132843}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":132843}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6809","nonce":132843}]},"public_addr":"172.21.15.83:6805/132843","cluster_addr":"172.21.15.83:6804/132843","heartbeat_back_addr":"172.21.15.83:6808/132843","heartbeat_front_addr":"172.21.15.83:6809/132843","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-04-30T19:06:56.619249+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":6,"length":2},{"begin":10,"length":1},{"begin":12,"length":3},{"begin":16,"length":1},{"begin":18,"length":1},{"begin":21,"length":1},{"begin":24,"length":2},{"begin":28,"length":1},{"begin":30,"length":3},{"begin":34,"length":1},{"begin":38,"length":3},{"begin":43,"length":1},{"begin":45,"length":2},{"begin":48,"length":2},{"begin":52,"length":2},{"begin":55,"length":2},{"begin":58,"length":1},{"begin":60,"length":3},{"begin":64,"length":1},{"begin":66,"length":1},{"begin":68,"length":4},{"begin":75,"length":2},{"begin":78,"length":1},{"begin":80,"length":3},{"begin":85,"length":1},{"begin":90,"length":1},{"begin":97,"length":1},{"begin":99,"length":1},{"begin":106,"length":1},{"begin":108,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-04-30T19:10:55.135 DEBUG:teuthology.orchestra.run.smithi033:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json Ignoring malparsed backtrace: 2023-04-30T19:11:38.383 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.2 is failed for ~265s , 2023-04-30T19:11:38.383 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~265s , 2023-04-30T19:11:38.383 INFO:tasks.daemonwatchdog.daemon_watchdog:BARK! unmounting mounts and killing all daemons Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_teuthology_69caab8ec6f44c61c991ae693e8e7b8cd859a896/teuthology/orchestra/run.py", line 479, in wait , proc.wait() , File "/home/teuthworker/src/git.ceph.com_teuthology_69caab8ec6f44c61c991ae693e8e7b8cd859a896/teuthology/orchestra/run.py", line 161, in wait Ignoring malparsed backtrace: 2023-04-30T20:03:39.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 , 2023-04-30T20:03:39.220 ERROR:teuthology.orchestra.daemon.state:Error while waiting for process to exit , Traceback (most recent call last): Ignoring malparsed backtrace: 2023-04-30T20:03:39.398 DEBUG:teuthology.orchestra.run.smithi033:> sync && sudo umount -f /var/lib/ceph/osd/ceph-1 , 2023-04-30T20:03:39.461 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-2 on ubuntu@smithi083.front.sepia.ceph.com , 2023-04-30T20:03:39.462 DEBUG:teuthology.orchestra.run.smithi083:> sync && sudo umount -f /var/lib/ceph/osd/ceph-2 Ignoring malparsed backtrace: 2023-04-30T20:03:43.072 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi033.front.sepia.ceph.com , 2023-04-30T20:03:43.073 DEBUG:teuthology.orchestra.run.smithi033:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 , 2023-04-30T20:03:43.097 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi083.front.sepia.ceph.com Ignoring malparsed backtrace: ceph version 18.0.0-3633-g0ae30b6b (0ae30b6b0a0fdd9fa3bc438c826208438ac27aa2) reef (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7fbb1a71bed4] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x1b0) [0x558a06a1d0e8] Ignoring malparsed backtrace: 2023-04-30T19:07:11.683+0000 7f1468f55700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-04-30T19:07:11.692+0000 7f1468f55700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , set osd.3 primary-affinity to 1 (8655362) Ignoring malparsed backtrace: 2023-04-30T19:07:53.367 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":142,"fsid":"748dd0c6-7174-4734-8b97-28004cdc5078","created":"2023-04-30T19:04:13.497850+0000","modified":"2023-04-30T19:07:12.533499+0000","last_up_change":"2023-04-30T19:06:56.619249+0000","last_in_change":"2023-04-30T19:04:15.788663+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:04:23.469100+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.75,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:04:24.177609+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:04:31.063009+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"134","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":116,"snap_epoch":134,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"d50c1949-1aed-45d6-b9f6-b8428fee01d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6801","nonce":133213}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":133213}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6805","nonce":133213}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":133213}]},"public_addr":"172.21.15.33:6801/133213","cluster_addr":"172.21.15.33:6800/133213","heartbeat_back_addr":"172.21.15.33:6805/133213","heartbeat_front_addr":"172.21.15.33:6804/133213","state":["exists","up"]},{"osd":1,"uuid":"1ae75fa1-248f-4f54-8b4a-4fa216c679a3","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":131,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6803","nonce":133240}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":133240}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":133240}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6807","nonce":133240}]},"public_addr":"172.21.15.33:6803/133240","cluster_addr":"172.21.15.33:6802/133240","heartbeat_back_addr":"172.21.15.33:6806/133240","heartbeat_front_addr":"172.21.15.33:6807/133240","state":["exists"]},{"osd":2,"uuid":"13c070d4-a782-403e-9494-13d35a1fd8ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6803","nonce":132825}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":132825}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":132825}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6807","nonce":132825}]},"public_addr":"172.21.15.83:6803/132825","cluster_addr":"172.21.15.83:6802/132825","heartbeat_back_addr":"172.21.15.83:6806/132825","heartbeat_front_addr":"172.21.15.83:6807/132825","state":["exists","up"]},{"osd":3,"uuid":"69ba14d8-63f6-4de7-83e9-9504e8192b8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6805","nonce":132843}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":132843}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":132843}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6809","nonce":132843}]},"public_addr":"172.21.15.83:6805/132843","cluster_addr":"172.21.15.83:6804/132843","heartbeat_back_addr":"172.21.15.83:6808/132843","heartbeat_front_addr":"172.21.15.83:6809/132843","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-04-30T19:06:56.619249+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":6,"length":2},{"begin":10,"length":1},{"begin":12,"length":3},{"begin":16,"length":1},{"begin":18,"length":1},{"begin":21,"length":1},{"begin":24,"length":2},{"begin":28,"length":1},{"begin":30,"length":3},{"begin":34,"length":1},{"begin":38,"length":3},{"begin":43,"length":1},{"begin":45,"length":2},{"begin":48,"length":2},{"begin":52,"length":2},{"begin":55,"length":2},{"begin":58,"length":1},{"begin":60,"length":3},{"begin":64,"length":1},{"begin":66,"length":1},{"begin":68,"length":4},{"begin":75,"length":2},{"begin":78,"length":1},{"begin":80,"length":3},{"begin":85,"length":1},{"begin":90,"length":1},{"begin":97,"length":1},{"begin":99,"length":1},{"begin":106,"length":1},{"begin":108,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-04-30T19:07:54.648 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~47s , 2023-04-30T19:07:54.648 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.1 is failed for ~80s Ignoring malparsed backtrace: 2023-04-30T19:08:37.213 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":142,"fsid":"748dd0c6-7174-4734-8b97-28004cdc5078","created":"2023-04-30T19:04:13.497850+0000","modified":"2023-04-30T19:07:12.533499+0000","last_up_change":"2023-04-30T19:06:56.619249+0000","last_in_change":"2023-04-30T19:04:15.788663+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:04:23.469100+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.75,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:04:24.177609+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:04:31.063009+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"134","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":116,"snap_epoch":134,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"d50c1949-1aed-45d6-b9f6-b8428fee01d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6801","nonce":133213}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":133213}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6805","nonce":133213}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":133213}]},"public_addr":"172.21.15.33:6801/133213","cluster_addr":"172.21.15.33:6800/133213","heartbeat_back_addr":"172.21.15.33:6805/133213","heartbeat_front_addr":"172.21.15.33:6804/133213","state":["exists","up"]},{"osd":1,"uuid":"1ae75fa1-248f-4f54-8b4a-4fa216c679a3","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":131,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6803","nonce":133240}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":133240}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":133240}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6807","nonce":133240}]},"public_addr":"172.21.15.33:6803/133240","cluster_addr":"172.21.15.33:6802/133240","heartbeat_back_addr":"172.21.15.33:6806/133240","heartbeat_front_addr":"172.21.15.33:6807/133240","state":["exists"]},{"osd":2,"uuid":"13c070d4-a782-403e-9494-13d35a1fd8ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6803","nonce":132825}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":132825}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":132825}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6807","nonce":132825}]},"public_addr":"172.21.15.83:6803/132825","cluster_addr":"172.21.15.83:6802/132825","heartbeat_back_addr":"172.21.15.83:6806/132825","heartbeat_front_addr":"172.21.15.83:6807/132825","state":["exists","up"]},{"osd":3,"uuid":"69ba14d8-63f6-4de7-83e9-9504e8192b8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6805","nonce":132843}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":132843}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":132843}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6809","nonce":132843}]},"public_addr":"172.21.15.83:6805/132843","cluster_addr":"172.21.15.83:6804/132843","heartbeat_back_addr":"172.21.15.83:6808/132843","heartbeat_front_addr":"172.21.15.83:6809/132843","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-04-30T19:06:56.619249+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":6,"length":2},{"begin":10,"length":1},{"begin":12,"length":3},{"begin":16,"length":1},{"begin":18,"length":1},{"begin":21,"length":1},{"begin":24,"length":2},{"begin":28,"length":1},{"begin":30,"length":3},{"begin":34,"length":1},{"begin":38,"length":3},{"begin":43,"length":1},{"begin":45,"length":2},{"begin":48,"length":2},{"begin":52,"length":2},{"begin":55,"length":2},{"begin":58,"length":1},{"begin":60,"length":3},{"begin":64,"length":1},{"begin":66,"length":1},{"begin":68,"length":4},{"begin":75,"length":2},{"begin":78,"length":1},{"begin":80,"length":3},{"begin":85,"length":1},{"begin":90,"length":1},{"begin":97,"length":1},{"begin":99,"length":1},{"begin":106,"length":1},{"begin":108,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-04-30T19:08:40.228 DEBUG:teuthology.orchestra.run.smithi033:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-04-30T19:08:40.374+0000 7f39e28d1700 -1 WARNING: the following dangerous and experimental features are enabled: crimson Ignoring malparsed backtrace: 2023-04-30T19:09:23.095 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~130s , 2023-04-30T19:09:24.075 DEBUG:teuthology.orchestra.run.smithi033:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-04-30T19:09:24.219+0000 7f655196c700 -1 WARNING: the following dangerous and experimental features are enabled: crimson Ignoring malparsed backtrace: 2023-04-30T19:10:08.280 INFO:teuthology.orchestra.run.smithi033.stdout: , 2023-04-30T19:10:08.280 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":142,"fsid":"748dd0c6-7174-4734-8b97-28004cdc5078","created":"2023-04-30T19:04:13.497850+0000","modified":"2023-04-30T19:07:12.533499+0000","last_up_change":"2023-04-30T19:06:56.619249+0000","last_in_change":"2023-04-30T19:04:15.788663+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:04:23.469100+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.75,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:04:24.177609+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:04:31.063009+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"134","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":116,"snap_epoch":134,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"d50c1949-1aed-45d6-b9f6-b8428fee01d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6801","nonce":133213}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":133213}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6805","nonce":133213}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":133213}]},"public_addr":"172.21.15.33:6801/133213","cluster_addr":"172.21.15.33:6800/133213","heartbeat_back_addr":"172.21.15.33:6805/133213","heartbeat_front_addr":"172.21.15.33:6804/133213","state":["exists","up"]},{"osd":1,"uuid":"1ae75fa1-248f-4f54-8b4a-4fa216c679a3","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":131,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6803","nonce":133240}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":133240}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":133240}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6807","nonce":133240}]},"public_addr":"172.21.15.33:6803/133240","cluster_addr":"172.21.15.33:6802/133240","heartbeat_back_addr":"172.21.15.33:6806/133240","heartbeat_front_addr":"172.21.15.33:6807/133240","state":["exists"]},{"osd":2,"uuid":"13c070d4-a782-403e-9494-13d35a1fd8ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6803","nonce":132825}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":132825}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":132825}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6807","nonce":132825}]},"public_addr":"172.21.15.83:6803/132825","cluster_addr":"172.21.15.83:6802/132825","heartbeat_back_addr":"172.21.15.83:6806/132825","heartbeat_front_addr":"172.21.15.83:6807/132825","state":["exists","up"]},{"osd":3,"uuid":"69ba14d8-63f6-4de7-83e9-9504e8192b8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6805","nonce":132843}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":132843}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":132843}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6809","nonce":132843}]},"public_addr":"172.21.15.83:6805/132843","cluster_addr":"172.21.15.83:6804/132843","heartbeat_back_addr":"172.21.15.83:6808/132843","heartbeat_front_addr":"172.21.15.83:6809/132843","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-04-30T19:06:56.619249+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":6,"length":2},{"begin":10,"length":1},{"begin":12,"length":3},{"begin":16,"length":1},{"begin":18,"length":1},{"begin":21,"length":1},{"begin":24,"length":2},{"begin":28,"length":1},{"begin":30,"length":3},{"begin":34,"length":1},{"begin":38,"length":3},{"begin":43,"length":1},{"begin":45,"length":2},{"begin":48,"length":2},{"begin":52,"length":2},{"begin":55,"length":2},{"begin":58,"length":1},{"begin":60,"length":3},{"begin":64,"length":1},{"begin":66,"length":1},{"begin":68,"length":4},{"begin":75,"length":2},{"begin":78,"length":1},{"begin":80,"length":3},{"begin":85,"length":1},{"begin":90,"length":1},{"begin":97,"length":1},{"begin":99,"length":1},{"begin":106,"length":1},{"begin":108,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-04-30T19:10:09.933 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~182s Ignoring malparsed backtrace: 2023-04-30T19:10:52.120 INFO:teuthology.orchestra.run.smithi033.stdout: , 2023-04-30T19:10:52.120 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":142,"fsid":"748dd0c6-7174-4734-8b97-28004cdc5078","created":"2023-04-30T19:04:13.497850+0000","modified":"2023-04-30T19:07:12.533499+0000","last_up_change":"2023-04-30T19:06:56.619249+0000","last_in_change":"2023-04-30T19:04:15.788663+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:04:23.469100+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.75,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:04:24.177609+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:04:31.063009+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"134","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":116,"snap_epoch":134,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"d50c1949-1aed-45d6-b9f6-b8428fee01d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6801","nonce":133213}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":133213}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6805","nonce":133213}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":133213}]},"public_addr":"172.21.15.33:6801/133213","cluster_addr":"172.21.15.33:6800/133213","heartbeat_back_addr":"172.21.15.33:6805/133213","heartbeat_front_addr":"172.21.15.33:6804/133213","state":["exists","up"]},{"osd":1,"uuid":"1ae75fa1-248f-4f54-8b4a-4fa216c679a3","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":131,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6803","nonce":133240}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":133240}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":133240}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6807","nonce":133240}]},"public_addr":"172.21.15.33:6803/133240","cluster_addr":"172.21.15.33:6802/133240","heartbeat_back_addr":"172.21.15.33:6806/133240","heartbeat_front_addr":"172.21.15.33:6807/133240","state":["exists"]},{"osd":2,"uuid":"13c070d4-a782-403e-9494-13d35a1fd8ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6803","nonce":132825}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":132825}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":132825}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6807","nonce":132825}]},"public_addr":"172.21.15.83:6803/132825","cluster_addr":"172.21.15.83:6802/132825","heartbeat_back_addr":"172.21.15.83:6806/132825","heartbeat_front_addr":"172.21.15.83:6807/132825","state":["exists","up"]},{"osd":3,"uuid":"69ba14d8-63f6-4de7-83e9-9504e8192b8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6805","nonce":132843}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":132843}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":132843}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6809","nonce":132843}]},"public_addr":"172.21.15.83:6805/132843","cluster_addr":"172.21.15.83:6804/132843","heartbeat_back_addr":"172.21.15.83:6808/132843","heartbeat_front_addr":"172.21.15.83:6809/132843","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-04-30T19:06:56.619249+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":6,"length":2},{"begin":10,"length":1},{"begin":12,"length":3},{"begin":16,"length":1},{"begin":18,"length":1},{"begin":21,"length":1},{"begin":24,"length":2},{"begin":28,"length":1},{"begin":30,"length":3},{"begin":34,"length":1},{"begin":38,"length":3},{"begin":43,"length":1},{"begin":45,"length":2},{"begin":48,"length":2},{"begin":52,"length":2},{"begin":55,"length":2},{"begin":58,"length":1},{"begin":60,"length":3},{"begin":64,"length":1},{"begin":66,"length":1},{"begin":68,"length":4},{"begin":75,"length":2},{"begin":78,"length":1},{"begin":80,"length":3},{"begin":85,"length":1},{"begin":90,"length":1},{"begin":97,"length":1},{"begin":99,"length":1},{"begin":106,"length":1},{"begin":108,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-04-30T19:10:55.135 DEBUG:teuthology.orchestra.run.smithi033:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json Ignoring malparsed backtrace: 2023-04-30T19:11:38.383 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.2 is failed for ~265s , 2023-04-30T19:11:38.383 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~265s , 2023-04-30T19:11:38.383 INFO:tasks.daemonwatchdog.daemon_watchdog:BARK! unmounting mounts and killing all daemons Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_teuthology_69caab8ec6f44c61c991ae693e8e7b8cd859a896/teuthology/orchestra/run.py", line 479, in wait , proc.wait() , File "/home/teuthworker/src/git.ceph.com_teuthology_69caab8ec6f44c61c991ae693e8e7b8cd859a896/teuthology/orchestra/run.py", line 161, in wait Ignoring malparsed backtrace: 2023-04-30T20:03:39.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 , 2023-04-30T20:03:39.220 ERROR:teuthology.orchestra.daemon.state:Error while waiting for process to exit , Traceback (most recent call last): Ignoring malparsed backtrace: 2023-04-30T20:03:39.398 DEBUG:teuthology.orchestra.run.smithi033:> sync && sudo umount -f /var/lib/ceph/osd/ceph-1 , 2023-04-30T20:03:39.461 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-2 on ubuntu@smithi083.front.sepia.ceph.com , 2023-04-30T20:03:39.462 DEBUG:teuthology.orchestra.run.smithi083:> sync && sudo umount -f /var/lib/ceph/osd/ceph-2 Ignoring malparsed backtrace: 2023-04-30T20:03:43.072 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi033.front.sepia.ceph.com , 2023-04-30T20:03:43.073 DEBUG:teuthology.orchestra.run.smithi033:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 , 2023-04-30T20:03:43.097 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi083.front.sepia.ceph.com Ignoring malparsed backtrace: ceph version 18.0.0-3633-g0ae30b6b (0ae30b6b0a0fdd9fa3bc438c826208438ac27aa2) reef (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7fbb1a71bed4] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x1b0) [0x558a06a1d0e8] Ignoring malparsed backtrace: 2023-04-30T19:07:11.683+0000 7f1468f55700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-04-30T19:07:11.692+0000 7f1468f55700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , set osd.3 primary-affinity to 1 (8655362) Ignoring malparsed backtrace: 2023-04-30T19:07:53.367 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":142,"fsid":"748dd0c6-7174-4734-8b97-28004cdc5078","created":"2023-04-30T19:04:13.497850+0000","modified":"2023-04-30T19:07:12.533499+0000","last_up_change":"2023-04-30T19:06:56.619249+0000","last_in_change":"2023-04-30T19:04:15.788663+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:04:23.469100+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.75,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:04:24.177609+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:04:31.063009+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"134","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":116,"snap_epoch":134,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"d50c1949-1aed-45d6-b9f6-b8428fee01d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6801","nonce":133213}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":133213}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6805","nonce":133213}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":133213}]},"public_addr":"172.21.15.33:6801/133213","cluster_addr":"172.21.15.33:6800/133213","heartbeat_back_addr":"172.21.15.33:6805/133213","heartbeat_front_addr":"172.21.15.33:6804/133213","state":["exists","up"]},{"osd":1,"uuid":"1ae75fa1-248f-4f54-8b4a-4fa216c679a3","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":131,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6803","nonce":133240}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":133240}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":133240}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6807","nonce":133240}]},"public_addr":"172.21.15.33:6803/133240","cluster_addr":"172.21.15.33:6802/133240","heartbeat_back_addr":"172.21.15.33:6806/133240","heartbeat_front_addr":"172.21.15.33:6807/133240","state":["exists"]},{"osd":2,"uuid":"13c070d4-a782-403e-9494-13d35a1fd8ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6803","nonce":132825}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":132825}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":132825}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6807","nonce":132825}]},"public_addr":"172.21.15.83:6803/132825","cluster_addr":"172.21.15.83:6802/132825","heartbeat_back_addr":"172.21.15.83:6806/132825","heartbeat_front_addr":"172.21.15.83:6807/132825","state":["exists","up"]},{"osd":3,"uuid":"69ba14d8-63f6-4de7-83e9-9504e8192b8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6805","nonce":132843}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":132843}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":132843}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6809","nonce":132843}]},"public_addr":"172.21.15.83:6805/132843","cluster_addr":"172.21.15.83:6804/132843","heartbeat_back_addr":"172.21.15.83:6808/132843","heartbeat_front_addr":"172.21.15.83:6809/132843","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-04-30T19:06:56.619249+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":6,"length":2},{"begin":10,"length":1},{"begin":12,"length":3},{"begin":16,"length":1},{"begin":18,"length":1},{"begin":21,"length":1},{"begin":24,"length":2},{"begin":28,"length":1},{"begin":30,"length":3},{"begin":34,"length":1},{"begin":38,"length":3},{"begin":43,"length":1},{"begin":45,"length":2},{"begin":48,"length":2},{"begin":52,"length":2},{"begin":55,"length":2},{"begin":58,"length":1},{"begin":60,"length":3},{"begin":64,"length":1},{"begin":66,"length":1},{"begin":68,"length":4},{"begin":75,"length":2},{"begin":78,"length":1},{"begin":80,"length":3},{"begin":85,"length":1},{"begin":90,"length":1},{"begin":97,"length":1},{"begin":99,"length":1},{"begin":106,"length":1},{"begin":108,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-04-30T19:07:54.648 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~47s , 2023-04-30T19:07:54.648 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.1 is failed for ~80s Ignoring malparsed backtrace: 2023-04-30T19:08:37.213 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":142,"fsid":"748dd0c6-7174-4734-8b97-28004cdc5078","created":"2023-04-30T19:04:13.497850+0000","modified":"2023-04-30T19:07:12.533499+0000","last_up_change":"2023-04-30T19:06:56.619249+0000","last_in_change":"2023-04-30T19:04:15.788663+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:04:23.469100+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.75,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:04:24.177609+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:04:31.063009+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"134","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":116,"snap_epoch":134,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"d50c1949-1aed-45d6-b9f6-b8428fee01d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6801","nonce":133213}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":133213}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6805","nonce":133213}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":133213}]},"public_addr":"172.21.15.33:6801/133213","cluster_addr":"172.21.15.33:6800/133213","heartbeat_back_addr":"172.21.15.33:6805/133213","heartbeat_front_addr":"172.21.15.33:6804/133213","state":["exists","up"]},{"osd":1,"uuid":"1ae75fa1-248f-4f54-8b4a-4fa216c679a3","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":131,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6803","nonce":133240}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":133240}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":133240}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6807","nonce":133240}]},"public_addr":"172.21.15.33:6803/133240","cluster_addr":"172.21.15.33:6802/133240","heartbeat_back_addr":"172.21.15.33:6806/133240","heartbeat_front_addr":"172.21.15.33:6807/133240","state":["exists"]},{"osd":2,"uuid":"13c070d4-a782-403e-9494-13d35a1fd8ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6803","nonce":132825}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":132825}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":132825}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6807","nonce":132825}]},"public_addr":"172.21.15.83:6803/132825","cluster_addr":"172.21.15.83:6802/132825","heartbeat_back_addr":"172.21.15.83:6806/132825","heartbeat_front_addr":"172.21.15.83:6807/132825","state":["exists","up"]},{"osd":3,"uuid":"69ba14d8-63f6-4de7-83e9-9504e8192b8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6805","nonce":132843}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":132843}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":132843}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6809","nonce":132843}]},"public_addr":"172.21.15.83:6805/132843","cluster_addr":"172.21.15.83:6804/132843","heartbeat_back_addr":"172.21.15.83:6808/132843","heartbeat_front_addr":"172.21.15.83:6809/132843","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-04-30T19:06:56.619249+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":6,"length":2},{"begin":10,"length":1},{"begin":12,"length":3},{"begin":16,"length":1},{"begin":18,"length":1},{"begin":21,"length":1},{"begin":24,"length":2},{"begin":28,"length":1},{"begin":30,"length":3},{"begin":34,"length":1},{"begin":38,"length":3},{"begin":43,"length":1},{"begin":45,"length":2},{"begin":48,"length":2},{"begin":52,"length":2},{"begin":55,"length":2},{"begin":58,"length":1},{"begin":60,"length":3},{"begin":64,"length":1},{"begin":66,"length":1},{"begin":68,"length":4},{"begin":75,"length":2},{"begin":78,"length":1},{"begin":80,"length":3},{"begin":85,"length":1},{"begin":90,"length":1},{"begin":97,"length":1},{"begin":99,"length":1},{"begin":106,"length":1},{"begin":108,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-04-30T19:08:40.228 DEBUG:teuthology.orchestra.run.smithi033:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-04-30T19:08:40.374+0000 7f39e28d1700 -1 WARNING: the following dangerous and experimental features are enabled: crimson Ignoring malparsed backtrace: 2023-04-30T19:09:23.095 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~130s , 2023-04-30T19:09:24.075 DEBUG:teuthology.orchestra.run.smithi033:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-04-30T19:09:24.219+0000 7f655196c700 -1 WARNING: the following dangerous and experimental features are enabled: crimson Ignoring malparsed backtrace: 2023-04-30T19:10:08.280 INFO:teuthology.orchestra.run.smithi033.stdout: , 2023-04-30T19:10:08.280 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":142,"fsid":"748dd0c6-7174-4734-8b97-28004cdc5078","created":"2023-04-30T19:04:13.497850+0000","modified":"2023-04-30T19:07:12.533499+0000","last_up_change":"2023-04-30T19:06:56.619249+0000","last_in_change":"2023-04-30T19:04:15.788663+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:04:23.469100+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.75,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:04:24.177609+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:04:31.063009+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"134","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":116,"snap_epoch":134,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"d50c1949-1aed-45d6-b9f6-b8428fee01d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6801","nonce":133213}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":133213}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6805","nonce":133213}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":133213}]},"public_addr":"172.21.15.33:6801/133213","cluster_addr":"172.21.15.33:6800/133213","heartbeat_back_addr":"172.21.15.33:6805/133213","heartbeat_front_addr":"172.21.15.33:6804/133213","state":["exists","up"]},{"osd":1,"uuid":"1ae75fa1-248f-4f54-8b4a-4fa216c679a3","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":131,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6803","nonce":133240}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":133240}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":133240}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6807","nonce":133240}]},"public_addr":"172.21.15.33:6803/133240","cluster_addr":"172.21.15.33:6802/133240","heartbeat_back_addr":"172.21.15.33:6806/133240","heartbeat_front_addr":"172.21.15.33:6807/133240","state":["exists"]},{"osd":2,"uuid":"13c070d4-a782-403e-9494-13d35a1fd8ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6803","nonce":132825}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":132825}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":132825}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6807","nonce":132825}]},"public_addr":"172.21.15.83:6803/132825","cluster_addr":"172.21.15.83:6802/132825","heartbeat_back_addr":"172.21.15.83:6806/132825","heartbeat_front_addr":"172.21.15.83:6807/132825","state":["exists","up"]},{"osd":3,"uuid":"69ba14d8-63f6-4de7-83e9-9504e8192b8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6805","nonce":132843}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":132843}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":132843}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6809","nonce":132843}]},"public_addr":"172.21.15.83:6805/132843","cluster_addr":"172.21.15.83:6804/132843","heartbeat_back_addr":"172.21.15.83:6808/132843","heartbeat_front_addr":"172.21.15.83:6809/132843","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-04-30T19:06:56.619249+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":6,"length":2},{"begin":10,"length":1},{"begin":12,"length":3},{"begin":16,"length":1},{"begin":18,"length":1},{"begin":21,"length":1},{"begin":24,"length":2},{"begin":28,"length":1},{"begin":30,"length":3},{"begin":34,"length":1},{"begin":38,"length":3},{"begin":43,"length":1},{"begin":45,"length":2},{"begin":48,"length":2},{"begin":52,"length":2},{"begin":55,"length":2},{"begin":58,"length":1},{"begin":60,"length":3},{"begin":64,"length":1},{"begin":66,"length":1},{"begin":68,"length":4},{"begin":75,"length":2},{"begin":78,"length":1},{"begin":80,"length":3},{"begin":85,"length":1},{"begin":90,"length":1},{"begin":97,"length":1},{"begin":99,"length":1},{"begin":106,"length":1},{"begin":108,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-04-30T19:10:09.933 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~182s Ignoring malparsed backtrace: 2023-04-30T19:10:52.120 INFO:teuthology.orchestra.run.smithi033.stdout: , 2023-04-30T19:10:52.120 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":142,"fsid":"748dd0c6-7174-4734-8b97-28004cdc5078","created":"2023-04-30T19:04:13.497850+0000","modified":"2023-04-30T19:07:12.533499+0000","last_up_change":"2023-04-30T19:06:56.619249+0000","last_in_change":"2023-04-30T19:04:15.788663+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:04:23.469100+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.75,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:04:24.177609+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:04:31.063009+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"134","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":116,"snap_epoch":134,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"d50c1949-1aed-45d6-b9f6-b8428fee01d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6801","nonce":133213}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":133213}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6805","nonce":133213}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":133213}]},"public_addr":"172.21.15.33:6801/133213","cluster_addr":"172.21.15.33:6800/133213","heartbeat_back_addr":"172.21.15.33:6805/133213","heartbeat_front_addr":"172.21.15.33:6804/133213","state":["exists","up"]},{"osd":1,"uuid":"1ae75fa1-248f-4f54-8b4a-4fa216c679a3","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":131,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6803","nonce":133240}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":133240}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":133240}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6807","nonce":133240}]},"public_addr":"172.21.15.33:6803/133240","cluster_addr":"172.21.15.33:6802/133240","heartbeat_back_addr":"172.21.15.33:6806/133240","heartbeat_front_addr":"172.21.15.33:6807/133240","state":["exists"]},{"osd":2,"uuid":"13c070d4-a782-403e-9494-13d35a1fd8ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6803","nonce":132825}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":132825}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":132825}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6807","nonce":132825}]},"public_addr":"172.21.15.83:6803/132825","cluster_addr":"172.21.15.83:6802/132825","heartbeat_back_addr":"172.21.15.83:6806/132825","heartbeat_front_addr":"172.21.15.83:6807/132825","state":["exists","up"]},{"osd":3,"uuid":"69ba14d8-63f6-4de7-83e9-9504e8192b8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6805","nonce":132843}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":132843}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":132843}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6809","nonce":132843}]},"public_addr":"172.21.15.83:6805/132843","cluster_addr":"172.21.15.83:6804/132843","heartbeat_back_addr":"172.21.15.83:6808/132843","heartbeat_front_addr":"172.21.15.83:6809/132843","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-04-30T19:06:56.619249+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":6,"length":2},{"begin":10,"length":1},{"begin":12,"length":3},{"begin":16,"length":1},{"begin":18,"length":1},{"begin":21,"length":1},{"begin":24,"length":2},{"begin":28,"length":1},{"begin":30,"length":3},{"begin":34,"length":1},{"begin":38,"length":3},{"begin":43,"length":1},{"begin":45,"length":2},{"begin":48,"length":2},{"begin":52,"length":2},{"begin":55,"length":2},{"begin":58,"length":1},{"begin":60,"length":3},{"begin":64,"length":1},{"begin":66,"length":1},{"begin":68,"length":4},{"begin":75,"length":2},{"begin":78,"length":1},{"begin":80,"length":3},{"begin":85,"length":1},{"begin":90,"length":1},{"begin":97,"length":1},{"begin":99,"length":1},{"begin":106,"length":1},{"begin":108,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-04-30T19:10:55.135 DEBUG:teuthology.orchestra.run.smithi033:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json Ignoring malparsed backtrace: 2023-04-30T19:11:38.383 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.2 is failed for ~265s , 2023-04-30T19:11:38.383 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~265s , 2023-04-30T19:11:38.383 INFO:tasks.daemonwatchdog.daemon_watchdog:BARK! unmounting mounts and killing all daemons Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_teuthology_69caab8ec6f44c61c991ae693e8e7b8cd859a896/teuthology/orchestra/run.py", line 479, in wait , proc.wait() , File "/home/teuthworker/src/git.ceph.com_teuthology_69caab8ec6f44c61c991ae693e8e7b8cd859a896/teuthology/orchestra/run.py", line 161, in wait Ignoring malparsed backtrace: 2023-04-30T20:03:39.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 , 2023-04-30T20:03:39.220 ERROR:teuthology.orchestra.daemon.state:Error while waiting for process to exit , Traceback (most recent call last): Ignoring malparsed backtrace: 2023-04-30T20:03:39.398 DEBUG:teuthology.orchestra.run.smithi033:> sync && sudo umount -f /var/lib/ceph/osd/ceph-1 , 2023-04-30T20:03:39.461 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-2 on ubuntu@smithi083.front.sepia.ceph.com , 2023-04-30T20:03:39.462 DEBUG:teuthology.orchestra.run.smithi083:> sync && sudo umount -f /var/lib/ceph/osd/ceph-2 Ignoring malparsed backtrace: 2023-04-30T20:03:43.072 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi033.front.sepia.ceph.com , 2023-04-30T20:03:43.073 DEBUG:teuthology.orchestra.run.smithi033:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 , 2023-04-30T20:03:43.097 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi083.front.sepia.ceph.com Ignoring malparsed backtrace: ceph version 18.0.0-3633-g0ae30b6b (0ae30b6b0a0fdd9fa3bc438c826208438ac27aa2) reef (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7fbb1a71bed4] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x1b0) [0x558a06a1d0e8] Ignoring malparsed backtrace: 2023-04-30T19:07:11.683+0000 7f1468f55700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-04-30T19:07:11.692+0000 7f1468f55700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , set osd.3 primary-affinity to 1 (8655362) Ignoring malparsed backtrace: 2023-04-30T19:07:53.367 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":142,"fsid":"748dd0c6-7174-4734-8b97-28004cdc5078","created":"2023-04-30T19:04:13.497850+0000","modified":"2023-04-30T19:07:12.533499+0000","last_up_change":"2023-04-30T19:06:56.619249+0000","last_in_change":"2023-04-30T19:04:15.788663+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:04:23.469100+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.75,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:04:24.177609+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:04:31.063009+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"134","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":116,"snap_epoch":134,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"d50c1949-1aed-45d6-b9f6-b8428fee01d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6801","nonce":133213}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":133213}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6805","nonce":133213}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":133213}]},"public_addr":"172.21.15.33:6801/133213","cluster_addr":"172.21.15.33:6800/133213","heartbeat_back_addr":"172.21.15.33:6805/133213","heartbeat_front_addr":"172.21.15.33:6804/133213","state":["exists","up"]},{"osd":1,"uuid":"1ae75fa1-248f-4f54-8b4a-4fa216c679a3","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":131,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6803","nonce":133240}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":133240}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":133240}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6807","nonce":133240}]},"public_addr":"172.21.15.33:6803/133240","cluster_addr":"172.21.15.33:6802/133240","heartbeat_back_addr":"172.21.15.33:6806/133240","heartbeat_front_addr":"172.21.15.33:6807/133240","state":["exists"]},{"osd":2,"uuid":"13c070d4-a782-403e-9494-13d35a1fd8ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6803","nonce":132825}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":132825}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":132825}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6807","nonce":132825}]},"public_addr":"172.21.15.83:6803/132825","cluster_addr":"172.21.15.83:6802/132825","heartbeat_back_addr":"172.21.15.83:6806/132825","heartbeat_front_addr":"172.21.15.83:6807/132825","state":["exists","up"]},{"osd":3,"uuid":"69ba14d8-63f6-4de7-83e9-9504e8192b8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6805","nonce":132843}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":132843}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":132843}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6809","nonce":132843}]},"public_addr":"172.21.15.83:6805/132843","cluster_addr":"172.21.15.83:6804/132843","heartbeat_back_addr":"172.21.15.83:6808/132843","heartbeat_front_addr":"172.21.15.83:6809/132843","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-04-30T19:06:56.619249+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":6,"length":2},{"begin":10,"length":1},{"begin":12,"length":3},{"begin":16,"length":1},{"begin":18,"length":1},{"begin":21,"length":1},{"begin":24,"length":2},{"begin":28,"length":1},{"begin":30,"length":3},{"begin":34,"length":1},{"begin":38,"length":3},{"begin":43,"length":1},{"begin":45,"length":2},{"begin":48,"length":2},{"begin":52,"length":2},{"begin":55,"length":2},{"begin":58,"length":1},{"begin":60,"length":3},{"begin":64,"length":1},{"begin":66,"length":1},{"begin":68,"length":4},{"begin":75,"length":2},{"begin":78,"length":1},{"begin":80,"length":3},{"begin":85,"length":1},{"begin":90,"length":1},{"begin":97,"length":1},{"begin":99,"length":1},{"begin":106,"length":1},{"begin":108,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-04-30T19:07:54.648 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~47s , 2023-04-30T19:07:54.648 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.1 is failed for ~80s Ignoring malparsed backtrace: 2023-04-30T19:08:37.213 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":142,"fsid":"748dd0c6-7174-4734-8b97-28004cdc5078","created":"2023-04-30T19:04:13.497850+0000","modified":"2023-04-30T19:07:12.533499+0000","last_up_change":"2023-04-30T19:06:56.619249+0000","last_in_change":"2023-04-30T19:04:15.788663+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:04:23.469100+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.75,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:04:24.177609+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:04:31.063009+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"134","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":116,"snap_epoch":134,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"d50c1949-1aed-45d6-b9f6-b8428fee01d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6801","nonce":133213}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":133213}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6805","nonce":133213}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":133213}]},"public_addr":"172.21.15.33:6801/133213","cluster_addr":"172.21.15.33:6800/133213","heartbeat_back_addr":"172.21.15.33:6805/133213","heartbeat_front_addr":"172.21.15.33:6804/133213","state":["exists","up"]},{"osd":1,"uuid":"1ae75fa1-248f-4f54-8b4a-4fa216c679a3","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":131,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6803","nonce":133240}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":133240}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":133240}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6807","nonce":133240}]},"public_addr":"172.21.15.33:6803/133240","cluster_addr":"172.21.15.33:6802/133240","heartbeat_back_addr":"172.21.15.33:6806/133240","heartbeat_front_addr":"172.21.15.33:6807/133240","state":["exists"]},{"osd":2,"uuid":"13c070d4-a782-403e-9494-13d35a1fd8ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6803","nonce":132825}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":132825}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":132825}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6807","nonce":132825}]},"public_addr":"172.21.15.83:6803/132825","cluster_addr":"172.21.15.83:6802/132825","heartbeat_back_addr":"172.21.15.83:6806/132825","heartbeat_front_addr":"172.21.15.83:6807/132825","state":["exists","up"]},{"osd":3,"uuid":"69ba14d8-63f6-4de7-83e9-9504e8192b8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6805","nonce":132843}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":132843}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":132843}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6809","nonce":132843}]},"public_addr":"172.21.15.83:6805/132843","cluster_addr":"172.21.15.83:6804/132843","heartbeat_back_addr":"172.21.15.83:6808/132843","heartbeat_front_addr":"172.21.15.83:6809/132843","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-04-30T19:06:56.619249+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":6,"length":2},{"begin":10,"length":1},{"begin":12,"length":3},{"begin":16,"length":1},{"begin":18,"length":1},{"begin":21,"length":1},{"begin":24,"length":2},{"begin":28,"length":1},{"begin":30,"length":3},{"begin":34,"length":1},{"begin":38,"length":3},{"begin":43,"length":1},{"begin":45,"length":2},{"begin":48,"length":2},{"begin":52,"length":2},{"begin":55,"length":2},{"begin":58,"length":1},{"begin":60,"length":3},{"begin":64,"length":1},{"begin":66,"length":1},{"begin":68,"length":4},{"begin":75,"length":2},{"begin":78,"length":1},{"begin":80,"length":3},{"begin":85,"length":1},{"begin":90,"length":1},{"begin":97,"length":1},{"begin":99,"length":1},{"begin":106,"length":1},{"begin":108,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-04-30T19:08:40.228 DEBUG:teuthology.orchestra.run.smithi033:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-04-30T19:08:40.374+0000 7f39e28d1700 -1 WARNING: the following dangerous and experimental features are enabled: crimson Ignoring malparsed backtrace: 2023-04-30T19:09:23.095 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~130s , 2023-04-30T19:09:24.075 DEBUG:teuthology.orchestra.run.smithi033:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-04-30T19:09:24.219+0000 7f655196c700 -1 WARNING: the following dangerous and experimental features are enabled: crimson Ignoring malparsed backtrace: 2023-04-30T19:10:08.280 INFO:teuthology.orchestra.run.smithi033.stdout: , 2023-04-30T19:10:08.280 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":142,"fsid":"748dd0c6-7174-4734-8b97-28004cdc5078","created":"2023-04-30T19:04:13.497850+0000","modified":"2023-04-30T19:07:12.533499+0000","last_up_change":"2023-04-30T19:06:56.619249+0000","last_in_change":"2023-04-30T19:04:15.788663+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:04:23.469100+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.75,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:04:24.177609+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:04:31.063009+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"134","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":116,"snap_epoch":134,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"d50c1949-1aed-45d6-b9f6-b8428fee01d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6801","nonce":133213}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":133213}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6805","nonce":133213}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":133213}]},"public_addr":"172.21.15.33:6801/133213","cluster_addr":"172.21.15.33:6800/133213","heartbeat_back_addr":"172.21.15.33:6805/133213","heartbeat_front_addr":"172.21.15.33:6804/133213","state":["exists","up"]},{"osd":1,"uuid":"1ae75fa1-248f-4f54-8b4a-4fa216c679a3","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":131,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6803","nonce":133240}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":133240}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":133240}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6807","nonce":133240}]},"public_addr":"172.21.15.33:6803/133240","cluster_addr":"172.21.15.33:6802/133240","heartbeat_back_addr":"172.21.15.33:6806/133240","heartbeat_front_addr":"172.21.15.33:6807/133240","state":["exists"]},{"osd":2,"uuid":"13c070d4-a782-403e-9494-13d35a1fd8ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6803","nonce":132825}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":132825}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":132825}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6807","nonce":132825}]},"public_addr":"172.21.15.83:6803/132825","cluster_addr":"172.21.15.83:6802/132825","heartbeat_back_addr":"172.21.15.83:6806/132825","heartbeat_front_addr":"172.21.15.83:6807/132825","state":["exists","up"]},{"osd":3,"uuid":"69ba14d8-63f6-4de7-83e9-9504e8192b8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6805","nonce":132843}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":132843}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":132843}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6809","nonce":132843}]},"public_addr":"172.21.15.83:6805/132843","cluster_addr":"172.21.15.83:6804/132843","heartbeat_back_addr":"172.21.15.83:6808/132843","heartbeat_front_addr":"172.21.15.83:6809/132843","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-04-30T19:06:56.619249+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":6,"length":2},{"begin":10,"length":1},{"begin":12,"length":3},{"begin":16,"length":1},{"begin":18,"length":1},{"begin":21,"length":1},{"begin":24,"length":2},{"begin":28,"length":1},{"begin":30,"length":3},{"begin":34,"length":1},{"begin":38,"length":3},{"begin":43,"length":1},{"begin":45,"length":2},{"begin":48,"length":2},{"begin":52,"length":2},{"begin":55,"length":2},{"begin":58,"length":1},{"begin":60,"length":3},{"begin":64,"length":1},{"begin":66,"length":1},{"begin":68,"length":4},{"begin":75,"length":2},{"begin":78,"length":1},{"begin":80,"length":3},{"begin":85,"length":1},{"begin":90,"length":1},{"begin":97,"length":1},{"begin":99,"length":1},{"begin":106,"length":1},{"begin":108,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-04-30T19:10:09.933 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~182s Ignoring malparsed backtrace: 2023-04-30T19:10:52.120 INFO:teuthology.orchestra.run.smithi033.stdout: , 2023-04-30T19:10:52.120 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":142,"fsid":"748dd0c6-7174-4734-8b97-28004cdc5078","created":"2023-04-30T19:04:13.497850+0000","modified":"2023-04-30T19:07:12.533499+0000","last_up_change":"2023-04-30T19:06:56.619249+0000","last_in_change":"2023-04-30T19:04:15.788663+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:04:23.469100+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.75,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:04:24.177609+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:04:31.063009+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"134","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":116,"snap_epoch":134,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"d50c1949-1aed-45d6-b9f6-b8428fee01d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6801","nonce":133213}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":133213}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6805","nonce":133213}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":133213}]},"public_addr":"172.21.15.33:6801/133213","cluster_addr":"172.21.15.33:6800/133213","heartbeat_back_addr":"172.21.15.33:6805/133213","heartbeat_front_addr":"172.21.15.33:6804/133213","state":["exists","up"]},{"osd":1,"uuid":"1ae75fa1-248f-4f54-8b4a-4fa216c679a3","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":131,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6803","nonce":133240}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":133240}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":133240}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6807","nonce":133240}]},"public_addr":"172.21.15.33:6803/133240","cluster_addr":"172.21.15.33:6802/133240","heartbeat_back_addr":"172.21.15.33:6806/133240","heartbeat_front_addr":"172.21.15.33:6807/133240","state":["exists"]},{"osd":2,"uuid":"13c070d4-a782-403e-9494-13d35a1fd8ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6803","nonce":132825}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":132825}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":132825}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6807","nonce":132825}]},"public_addr":"172.21.15.83:6803/132825","cluster_addr":"172.21.15.83:6802/132825","heartbeat_back_addr":"172.21.15.83:6806/132825","heartbeat_front_addr":"172.21.15.83:6807/132825","state":["exists","up"]},{"osd":3,"uuid":"69ba14d8-63f6-4de7-83e9-9504e8192b8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6805","nonce":132843}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":132843}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":132843}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6809","nonce":132843}]},"public_addr":"172.21.15.83:6805/132843","cluster_addr":"172.21.15.83:6804/132843","heartbeat_back_addr":"172.21.15.83:6808/132843","heartbeat_front_addr":"172.21.15.83:6809/132843","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-04-30T19:06:56.619249+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":6,"length":2},{"begin":10,"length":1},{"begin":12,"length":3},{"begin":16,"length":1},{"begin":18,"length":1},{"begin":21,"length":1},{"begin":24,"length":2},{"begin":28,"length":1},{"begin":30,"length":3},{"begin":34,"length":1},{"begin":38,"length":3},{"begin":43,"length":1},{"begin":45,"length":2},{"begin":48,"length":2},{"begin":52,"length":2},{"begin":55,"length":2},{"begin":58,"length":1},{"begin":60,"length":3},{"begin":64,"length":1},{"begin":66,"length":1},{"begin":68,"length":4},{"begin":75,"length":2},{"begin":78,"length":1},{"begin":80,"length":3},{"begin":85,"length":1},{"begin":90,"length":1},{"begin":97,"length":1},{"begin":99,"length":1},{"begin":106,"length":1},{"begin":108,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-04-30T19:10:55.135 DEBUG:teuthology.orchestra.run.smithi033:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json Ignoring malparsed backtrace: 2023-04-30T19:11:38.383 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.2 is failed for ~265s , 2023-04-30T19:11:38.383 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~265s , 2023-04-30T19:11:38.383 INFO:tasks.daemonwatchdog.daemon_watchdog:BARK! unmounting mounts and killing all daemons Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_teuthology_69caab8ec6f44c61c991ae693e8e7b8cd859a896/teuthology/orchestra/run.py", line 479, in wait , proc.wait() , File "/home/teuthworker/src/git.ceph.com_teuthology_69caab8ec6f44c61c991ae693e8e7b8cd859a896/teuthology/orchestra/run.py", line 161, in wait Ignoring malparsed backtrace: 2023-04-30T20:03:39.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 , 2023-04-30T20:03:39.220 ERROR:teuthology.orchestra.daemon.state:Error while waiting for process to exit , Traceback (most recent call last): Ignoring malparsed backtrace: 2023-04-30T20:03:39.398 DEBUG:teuthology.orchestra.run.smithi033:> sync && sudo umount -f /var/lib/ceph/osd/ceph-1 , 2023-04-30T20:03:39.461 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-2 on ubuntu@smithi083.front.sepia.ceph.com , 2023-04-30T20:03:39.462 DEBUG:teuthology.orchestra.run.smithi083:> sync && sudo umount -f /var/lib/ceph/osd/ceph-2 Ignoring malparsed backtrace: 2023-04-30T20:03:43.072 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi033.front.sepia.ceph.com , 2023-04-30T20:03:43.073 DEBUG:teuthology.orchestra.run.smithi033:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 , 2023-04-30T20:03:43.097 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi083.front.sepia.ceph.com Ignoring malparsed backtrace: ceph version 18.0.0-3633-g0ae30b6b (0ae30b6b0a0fdd9fa3bc438c826208438ac27aa2) reef (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7fbb1a71bed4] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x1b0) [0x558a06a1d0e8] Ignoring malparsed backtrace: 2023-04-30T19:07:11.683+0000 7f1468f55700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-04-30T19:07:11.692+0000 7f1468f55700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , set osd.3 primary-affinity to 1 (8655362) Ignoring malparsed backtrace: 2023-04-30T19:07:53.367 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":142,"fsid":"748dd0c6-7174-4734-8b97-28004cdc5078","created":"2023-04-30T19:04:13.497850+0000","modified":"2023-04-30T19:07:12.533499+0000","last_up_change":"2023-04-30T19:06:56.619249+0000","last_in_change":"2023-04-30T19:04:15.788663+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:04:23.469100+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.75,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:04:24.177609+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:04:31.063009+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"134","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":116,"snap_epoch":134,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"d50c1949-1aed-45d6-b9f6-b8428fee01d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6801","nonce":133213}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":133213}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6805","nonce":133213}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":133213}]},"public_addr":"172.21.15.33:6801/133213","cluster_addr":"172.21.15.33:6800/133213","heartbeat_back_addr":"172.21.15.33:6805/133213","heartbeat_front_addr":"172.21.15.33:6804/133213","state":["exists","up"]},{"osd":1,"uuid":"1ae75fa1-248f-4f54-8b4a-4fa216c679a3","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":131,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6803","nonce":133240}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":133240}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":133240}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6807","nonce":133240}]},"public_addr":"172.21.15.33:6803/133240","cluster_addr":"172.21.15.33:6802/133240","heartbeat_back_addr":"172.21.15.33:6806/133240","heartbeat_front_addr":"172.21.15.33:6807/133240","state":["exists"]},{"osd":2,"uuid":"13c070d4-a782-403e-9494-13d35a1fd8ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6803","nonce":132825}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":132825}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":132825}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6807","nonce":132825}]},"public_addr":"172.21.15.83:6803/132825","cluster_addr":"172.21.15.83:6802/132825","heartbeat_back_addr":"172.21.15.83:6806/132825","heartbeat_front_addr":"172.21.15.83:6807/132825","state":["exists","up"]},{"osd":3,"uuid":"69ba14d8-63f6-4de7-83e9-9504e8192b8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6805","nonce":132843}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":132843}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":132843}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6809","nonce":132843}]},"public_addr":"172.21.15.83:6805/132843","cluster_addr":"172.21.15.83:6804/132843","heartbeat_back_addr":"172.21.15.83:6808/132843","heartbeat_front_addr":"172.21.15.83:6809/132843","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-04-30T19:06:56.619249+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":6,"length":2},{"begin":10,"length":1},{"begin":12,"length":3},{"begin":16,"length":1},{"begin":18,"length":1},{"begin":21,"length":1},{"begin":24,"length":2},{"begin":28,"length":1},{"begin":30,"length":3},{"begin":34,"length":1},{"begin":38,"length":3},{"begin":43,"length":1},{"begin":45,"length":2},{"begin":48,"length":2},{"begin":52,"length":2},{"begin":55,"length":2},{"begin":58,"length":1},{"begin":60,"length":3},{"begin":64,"length":1},{"begin":66,"length":1},{"begin":68,"length":4},{"begin":75,"length":2},{"begin":78,"length":1},{"begin":80,"length":3},{"begin":85,"length":1},{"begin":90,"length":1},{"begin":97,"length":1},{"begin":99,"length":1},{"begin":106,"length":1},{"begin":108,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-04-30T19:07:54.648 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~47s , 2023-04-30T19:07:54.648 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.1 is failed for ~80s Ignoring malparsed backtrace: 2023-04-30T19:08:37.213 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":142,"fsid":"748dd0c6-7174-4734-8b97-28004cdc5078","created":"2023-04-30T19:04:13.497850+0000","modified":"2023-04-30T19:07:12.533499+0000","last_up_change":"2023-04-30T19:06:56.619249+0000","last_in_change":"2023-04-30T19:04:15.788663+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:04:23.469100+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.75,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:04:24.177609+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:04:31.063009+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"134","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":116,"snap_epoch":134,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"d50c1949-1aed-45d6-b9f6-b8428fee01d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6801","nonce":133213}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":133213}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6805","nonce":133213}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":133213}]},"public_addr":"172.21.15.33:6801/133213","cluster_addr":"172.21.15.33:6800/133213","heartbeat_back_addr":"172.21.15.33:6805/133213","heartbeat_front_addr":"172.21.15.33:6804/133213","state":["exists","up"]},{"osd":1,"uuid":"1ae75fa1-248f-4f54-8b4a-4fa216c679a3","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":131,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6803","nonce":133240}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":133240}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":133240}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6807","nonce":133240}]},"public_addr":"172.21.15.33:6803/133240","cluster_addr":"172.21.15.33:6802/133240","heartbeat_back_addr":"172.21.15.33:6806/133240","heartbeat_front_addr":"172.21.15.33:6807/133240","state":["exists"]},{"osd":2,"uuid":"13c070d4-a782-403e-9494-13d35a1fd8ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6803","nonce":132825}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":132825}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":132825}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6807","nonce":132825}]},"public_addr":"172.21.15.83:6803/132825","cluster_addr":"172.21.15.83:6802/132825","heartbeat_back_addr":"172.21.15.83:6806/132825","heartbeat_front_addr":"172.21.15.83:6807/132825","state":["exists","up"]},{"osd":3,"uuid":"69ba14d8-63f6-4de7-83e9-9504e8192b8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6805","nonce":132843}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":132843}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":132843}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6809","nonce":132843}]},"public_addr":"172.21.15.83:6805/132843","cluster_addr":"172.21.15.83:6804/132843","heartbeat_back_addr":"172.21.15.83:6808/132843","heartbeat_front_addr":"172.21.15.83:6809/132843","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-04-30T19:06:56.619249+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":6,"length":2},{"begin":10,"length":1},{"begin":12,"length":3},{"begin":16,"length":1},{"begin":18,"length":1},{"begin":21,"length":1},{"begin":24,"length":2},{"begin":28,"length":1},{"begin":30,"length":3},{"begin":34,"length":1},{"begin":38,"length":3},{"begin":43,"length":1},{"begin":45,"length":2},{"begin":48,"length":2},{"begin":52,"length":2},{"begin":55,"length":2},{"begin":58,"length":1},{"begin":60,"length":3},{"begin":64,"length":1},{"begin":66,"length":1},{"begin":68,"length":4},{"begin":75,"length":2},{"begin":78,"length":1},{"begin":80,"length":3},{"begin":85,"length":1},{"begin":90,"length":1},{"begin":97,"length":1},{"begin":99,"length":1},{"begin":106,"length":1},{"begin":108,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-04-30T19:08:40.228 DEBUG:teuthology.orchestra.run.smithi033:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-04-30T19:08:40.374+0000 7f39e28d1700 -1 WARNING: the following dangerous and experimental features are enabled: crimson Ignoring malparsed backtrace: 2023-04-30T19:09:23.095 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~130s , 2023-04-30T19:09:24.075 DEBUG:teuthology.orchestra.run.smithi033:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-04-30T19:09:24.219+0000 7f655196c700 -1 WARNING: the following dangerous and experimental features are enabled: crimson Ignoring malparsed backtrace: 2023-04-30T19:10:08.280 INFO:teuthology.orchestra.run.smithi033.stdout: , 2023-04-30T19:10:08.280 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":142,"fsid":"748dd0c6-7174-4734-8b97-28004cdc5078","created":"2023-04-30T19:04:13.497850+0000","modified":"2023-04-30T19:07:12.533499+0000","last_up_change":"2023-04-30T19:06:56.619249+0000","last_in_change":"2023-04-30T19:04:15.788663+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:04:23.469100+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.75,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:04:24.177609+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:04:31.063009+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"134","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":116,"snap_epoch":134,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"d50c1949-1aed-45d6-b9f6-b8428fee01d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6801","nonce":133213}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":133213}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6805","nonce":133213}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":133213}]},"public_addr":"172.21.15.33:6801/133213","cluster_addr":"172.21.15.33:6800/133213","heartbeat_back_addr":"172.21.15.33:6805/133213","heartbeat_front_addr":"172.21.15.33:6804/133213","state":["exists","up"]},{"osd":1,"uuid":"1ae75fa1-248f-4f54-8b4a-4fa216c679a3","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":131,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6803","nonce":133240}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":133240}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":133240}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6807","nonce":133240}]},"public_addr":"172.21.15.33:6803/133240","cluster_addr":"172.21.15.33:6802/133240","heartbeat_back_addr":"172.21.15.33:6806/133240","heartbeat_front_addr":"172.21.15.33:6807/133240","state":["exists"]},{"osd":2,"uuid":"13c070d4-a782-403e-9494-13d35a1fd8ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6803","nonce":132825}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":132825}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":132825}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6807","nonce":132825}]},"public_addr":"172.21.15.83:6803/132825","cluster_addr":"172.21.15.83:6802/132825","heartbeat_back_addr":"172.21.15.83:6806/132825","heartbeat_front_addr":"172.21.15.83:6807/132825","state":["exists","up"]},{"osd":3,"uuid":"69ba14d8-63f6-4de7-83e9-9504e8192b8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6805","nonce":132843}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":132843}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":132843}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6809","nonce":132843}]},"public_addr":"172.21.15.83:6805/132843","cluster_addr":"172.21.15.83:6804/132843","heartbeat_back_addr":"172.21.15.83:6808/132843","heartbeat_front_addr":"172.21.15.83:6809/132843","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-04-30T19:06:56.619249+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":6,"length":2},{"begin":10,"length":1},{"begin":12,"length":3},{"begin":16,"length":1},{"begin":18,"length":1},{"begin":21,"length":1},{"begin":24,"length":2},{"begin":28,"length":1},{"begin":30,"length":3},{"begin":34,"length":1},{"begin":38,"length":3},{"begin":43,"length":1},{"begin":45,"length":2},{"begin":48,"length":2},{"begin":52,"length":2},{"begin":55,"length":2},{"begin":58,"length":1},{"begin":60,"length":3},{"begin":64,"length":1},{"begin":66,"length":1},{"begin":68,"length":4},{"begin":75,"length":2},{"begin":78,"length":1},{"begin":80,"length":3},{"begin":85,"length":1},{"begin":90,"length":1},{"begin":97,"length":1},{"begin":99,"length":1},{"begin":106,"length":1},{"begin":108,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-04-30T19:10:09.933 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~182s Ignoring malparsed backtrace: 2023-04-30T19:10:52.120 INFO:teuthology.orchestra.run.smithi033.stdout: , 2023-04-30T19:10:52.120 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":142,"fsid":"748dd0c6-7174-4734-8b97-28004cdc5078","created":"2023-04-30T19:04:13.497850+0000","modified":"2023-04-30T19:07:12.533499+0000","last_up_change":"2023-04-30T19:06:56.619249+0000","last_in_change":"2023-04-30T19:04:15.788663+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:04:23.469100+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.75,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:04:24.177609+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:04:31.063009+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"134","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":116,"snap_epoch":134,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"d50c1949-1aed-45d6-b9f6-b8428fee01d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6801","nonce":133213}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":133213}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6805","nonce":133213}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":133213}]},"public_addr":"172.21.15.33:6801/133213","cluster_addr":"172.21.15.33:6800/133213","heartbeat_back_addr":"172.21.15.33:6805/133213","heartbeat_front_addr":"172.21.15.33:6804/133213","state":["exists","up"]},{"osd":1,"uuid":"1ae75fa1-248f-4f54-8b4a-4fa216c679a3","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":131,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6803","nonce":133240}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":133240}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":133240}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6807","nonce":133240}]},"public_addr":"172.21.15.33:6803/133240","cluster_addr":"172.21.15.33:6802/133240","heartbeat_back_addr":"172.21.15.33:6806/133240","heartbeat_front_addr":"172.21.15.33:6807/133240","state":["exists"]},{"osd":2,"uuid":"13c070d4-a782-403e-9494-13d35a1fd8ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6803","nonce":132825}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":132825}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":132825}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6807","nonce":132825}]},"public_addr":"172.21.15.83:6803/132825","cluster_addr":"172.21.15.83:6802/132825","heartbeat_back_addr":"172.21.15.83:6806/132825","heartbeat_front_addr":"172.21.15.83:6807/132825","state":["exists","up"]},{"osd":3,"uuid":"69ba14d8-63f6-4de7-83e9-9504e8192b8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6805","nonce":132843}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":132843}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":132843}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6809","nonce":132843}]},"public_addr":"172.21.15.83:6805/132843","cluster_addr":"172.21.15.83:6804/132843","heartbeat_back_addr":"172.21.15.83:6808/132843","heartbeat_front_addr":"172.21.15.83:6809/132843","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-04-30T19:06:56.619249+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":6,"length":2},{"begin":10,"length":1},{"begin":12,"length":3},{"begin":16,"length":1},{"begin":18,"length":1},{"begin":21,"length":1},{"begin":24,"length":2},{"begin":28,"length":1},{"begin":30,"length":3},{"begin":34,"length":1},{"begin":38,"length":3},{"begin":43,"length":1},{"begin":45,"length":2},{"begin":48,"length":2},{"begin":52,"length":2},{"begin":55,"length":2},{"begin":58,"length":1},{"begin":60,"length":3},{"begin":64,"length":1},{"begin":66,"length":1},{"begin":68,"length":4},{"begin":75,"length":2},{"begin":78,"length":1},{"begin":80,"length":3},{"begin":85,"length":1},{"begin":90,"length":1},{"begin":97,"length":1},{"begin":99,"length":1},{"begin":106,"length":1},{"begin":108,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-04-30T19:10:55.135 DEBUG:teuthology.orchestra.run.smithi033:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json Ignoring malparsed backtrace: 2023-04-30T19:11:38.383 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.2 is failed for ~265s , 2023-04-30T19:11:38.383 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~265s , 2023-04-30T19:11:38.383 INFO:tasks.daemonwatchdog.daemon_watchdog:BARK! unmounting mounts and killing all daemons Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_teuthology_69caab8ec6f44c61c991ae693e8e7b8cd859a896/teuthology/orchestra/run.py", line 479, in wait , proc.wait() , File "/home/teuthworker/src/git.ceph.com_teuthology_69caab8ec6f44c61c991ae693e8e7b8cd859a896/teuthology/orchestra/run.py", line 161, in wait Ignoring malparsed backtrace: 2023-04-30T20:03:39.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 , 2023-04-30T20:03:39.220 ERROR:teuthology.orchestra.daemon.state:Error while waiting for process to exit , Traceback (most recent call last): Ignoring malparsed backtrace: 2023-04-30T20:03:39.398 DEBUG:teuthology.orchestra.run.smithi033:> sync && sudo umount -f /var/lib/ceph/osd/ceph-1 , 2023-04-30T20:03:39.461 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-2 on ubuntu@smithi083.front.sepia.ceph.com , 2023-04-30T20:03:39.462 DEBUG:teuthology.orchestra.run.smithi083:> sync && sudo umount -f /var/lib/ceph/osd/ceph-2 Ignoring malparsed backtrace: 2023-04-30T20:03:43.072 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi033.front.sepia.ceph.com , 2023-04-30T20:03:43.073 DEBUG:teuthology.orchestra.run.smithi033:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 , 2023-04-30T20:03:43.097 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi083.front.sepia.ceph.com Ignoring malparsed backtrace: ceph version 18.0.0-3633-g0ae30b6b (0ae30b6b0a0fdd9fa3bc438c826208438ac27aa2) reef (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7f210aeeeed4] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x1b0) [0x55d571d6c0e8] Ignoring malparsed backtrace: 2023-04-30T19:14:57.734 DEBUG:teuthology.orchestra.run.smithi029:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd reweight 0 1 , 2023-04-30T19:14:57.864+0000 7f44382f7700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-04-30T19:14:57.873+0000 7f44382f7700 -1 WARNING: the following dangerous and experimental features are enabled: crimson Ignoring malparsed backtrace: 2023-04-30T19:15:39.693+0000 7f3b58d61700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-04-30T19:15:39.903 INFO:teuthology.orchestra.run.smithi029.stdout: , 2023-04-30T19:15:39.903 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":176,"fsid":"ceaeeece-22e2-48ff-9295-38245c7f7da2","created":"2023-04-30T19:11:42.428777+0000","modified":"2023-04-30T19:15:26.679433+0000","last_up_change":"2023-04-30T19:15:25.673717+0000","last_in_change":"2023-04-30T19:11:44.667369+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:11:52.626169+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:11:53.007000+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.5,"raw_score_acting":1,"raw_score_stable":1,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:11:59.880342+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"161","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":142,"snap_epoch":161,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":2.2599999904632568,"score_stable":2.2599999904632568,"optimal_score":0.5,"raw_score_acting":1.1299999952316284,"raw_score_stable":1.1299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"e35756d0-679f-4288-82f8-700d9c05f7de","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":175,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6801","nonce":133225}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6800","nonce":133225}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6805","nonce":133225}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":133225}]},"public_addr":"172.21.15.29:6801/133225","cluster_addr":"172.21.15.29:6800/133225","heartbeat_back_addr":"172.21.15.29:6805/133225","heartbeat_front_addr":"172.21.15.29:6804/133225","state":["exists","up"]},{"osd":1,"uuid":"a63fbdcb-d6de-40c4-9079-ea506fa562a2","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":159,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6803","nonce":133252}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":133252}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":133252}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6807","nonce":133252}]},"public_addr":"172.21.15.29:6803/133252","cluster_addr":"172.21.15.29:6802/133252","heartbeat_back_addr":"172.21.15.29:6806/133252","heartbeat_front_addr":"172.21.15.29:6807/133252","state":["exists"]},{"osd":2,"uuid":"65457eae-1cda-486f-9c35-8f806493219f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":175,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6801","nonce":132894}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6800","nonce":132894}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6807","nonce":132894}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6806","nonce":132894}]},"public_addr":"172.21.15.119:6801/132894","cluster_addr":"172.21.15.119:6800/132894","heartbeat_back_addr":"172.21.15.119:6807/132894","heartbeat_front_addr":"172.21.15.119:6806/132894","state":["exists","up"]},{"osd":3,"uuid":"351890ba-3634-423c-8e7b-7b84ac1438db","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":159,"down_at":175,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6802","nonce":132923}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6803","nonce":132923}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6808","nonce":132923}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6809","nonce":132923}]},"public_addr":"172.21.15.119:6802/132923","cluster_addr":"172.21.15.119:6803/132923","heartbeat_back_addr":"172.21.15.119:6808/132923","heartbeat_front_addr":"172.21.15.119:6809/132923","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-04-30T19:14:44.444308+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2023-04-30T19:15:25.673717+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":1},{"begin":8,"length":3},{"begin":14,"length":1},{"begin":16,"length":2},{"begin":20,"length":1},{"begin":22,"length":4},{"begin":27,"length":1},{"begin":31,"length":2},{"begin":36,"length":3},{"begin":40,"length":1},{"begin":42,"length":2},{"begin":45,"length":1},{"begin":49,"length":1},{"begin":51,"length":2},{"begin":55,"length":1},{"begin":58,"length":2},{"begin":61,"length":2},{"begin":64,"length":1},{"begin":68,"length":5},{"begin":76,"length":1},{"begin":78,"length":4},{"begin":83,"length":1},{"begin":87,"length":1},{"begin":90,"length":1},{"begin":92,"length":2},{"begin":95,"length":6},{"begin":103,"length":2},{"begin":107,"length":1},{"begin":109,"length":1},{"begin":112,"length":2},{"begin":115,"length":2},{"begin":122,"length":2},{"begin":125,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} Ignoring malparsed backtrace: 2023-04-30T19:16:33.944 INFO:teuthology.orchestra.run.smithi029.stdout: , 2023-04-30T19:16:33.945 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":176,"fsid":"ceaeeece-22e2-48ff-9295-38245c7f7da2","created":"2023-04-30T19:11:42.428777+0000","modified":"2023-04-30T19:15:26.679433+0000","last_up_change":"2023-04-30T19:15:25.673717+0000","last_in_change":"2023-04-30T19:11:44.667369+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:11:52.626169+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:11:53.007000+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.5,"raw_score_acting":1,"raw_score_stable":1,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:11:59.880342+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"161","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":142,"snap_epoch":161,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":2.2599999904632568,"score_stable":2.2599999904632568,"optimal_score":0.5,"raw_score_acting":1.1299999952316284,"raw_score_stable":1.1299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"e35756d0-679f-4288-82f8-700d9c05f7de","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":175,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6801","nonce":133225}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6800","nonce":133225}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6805","nonce":133225}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":133225}]},"public_addr":"172.21.15.29:6801/133225","cluster_addr":"172.21.15.29:6800/133225","heartbeat_back_addr":"172.21.15.29:6805/133225","heartbeat_front_addr":"172.21.15.29:6804/133225","state":["exists","up"]},{"osd":1,"uuid":"a63fbdcb-d6de-40c4-9079-ea506fa562a2","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":159,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6803","nonce":133252}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":133252}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":133252}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6807","nonce":133252}]},"public_addr":"172.21.15.29:6803/133252","cluster_addr":"172.21.15.29:6802/133252","heartbeat_back_addr":"172.21.15.29:6806/133252","heartbeat_front_addr":"172.21.15.29:6807/133252","state":["exists"]},{"osd":2,"uuid":"65457eae-1cda-486f-9c35-8f806493219f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":175,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6801","nonce":132894}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6800","nonce":132894}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6807","nonce":132894}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6806","nonce":132894}]},"public_addr":"172.21.15.119:6801/132894","cluster_addr":"172.21.15.119:6800/132894","heartbeat_back_addr":"172.21.15.119:6807/132894","heartbeat_front_addr":"172.21.15.119:6806/132894","state":["exists","up"]},{"osd":3,"uuid":"351890ba-3634-423c-8e7b-7b84ac1438db","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":159,"down_at":175,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6802","nonce":132923}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6803","nonce":132923}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6808","nonce":132923}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6809","nonce":132923}]},"public_addr":"172.21.15.119:6802/132923","cluster_addr":"172.21.15.119:6803/132923","heartbeat_back_addr":"172.21.15.119:6808/132923","heartbeat_front_addr":"172.21.15.119:6809/132923","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-04-30T19:14:44.444308+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2023-04-30T19:15:25.673717+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":1},{"begin":8,"length":3},{"begin":14,"length":1},{"begin":16,"length":2},{"begin":20,"length":1},{"begin":22,"length":4},{"begin":27,"length":1},{"begin":31,"length":2},{"begin":36,"length":3},{"begin":40,"length":1},{"begin":42,"length":2},{"begin":45,"length":1},{"begin":49,"length":1},{"begin":51,"length":2},{"begin":55,"length":1},{"begin":58,"length":2},{"begin":61,"length":2},{"begin":64,"length":1},{"begin":68,"length":5},{"begin":76,"length":1},{"begin":78,"length":4},{"begin":83,"length":1},{"begin":87,"length":1},{"begin":90,"length":1},{"begin":92,"length":2},{"begin":95,"length":6},{"begin":103,"length":2},{"begin":107,"length":1},{"begin":109,"length":1},{"begin":112,"length":2},{"begin":115,"length":2},{"begin":122,"length":2},{"begin":125,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-04-30T19:16:36.447 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.1 is failed for ~136s Ignoring malparsed backtrace: 2023-04-30T19:17:27.943 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":176,"fsid":"ceaeeece-22e2-48ff-9295-38245c7f7da2","created":"2023-04-30T19:11:42.428777+0000","modified":"2023-04-30T19:15:26.679433+0000","last_up_change":"2023-04-30T19:15:25.673717+0000","last_in_change":"2023-04-30T19:11:44.667369+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:11:52.626169+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:11:53.007000+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.5,"raw_score_acting":1,"raw_score_stable":1,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:11:59.880342+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"161","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":142,"snap_epoch":161,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":2.2599999904632568,"score_stable":2.2599999904632568,"optimal_score":0.5,"raw_score_acting":1.1299999952316284,"raw_score_stable":1.1299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"e35756d0-679f-4288-82f8-700d9c05f7de","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":175,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6801","nonce":133225}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6800","nonce":133225}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6805","nonce":133225}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":133225}]},"public_addr":"172.21.15.29:6801/133225","cluster_addr":"172.21.15.29:6800/133225","heartbeat_back_addr":"172.21.15.29:6805/133225","heartbeat_front_addr":"172.21.15.29:6804/133225","state":["exists","up"]},{"osd":1,"uuid":"a63fbdcb-d6de-40c4-9079-ea506fa562a2","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":159,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6803","nonce":133252}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":133252}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":133252}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6807","nonce":133252}]},"public_addr":"172.21.15.29:6803/133252","cluster_addr":"172.21.15.29:6802/133252","heartbeat_back_addr":"172.21.15.29:6806/133252","heartbeat_front_addr":"172.21.15.29:6807/133252","state":["exists"]},{"osd":2,"uuid":"65457eae-1cda-486f-9c35-8f806493219f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":175,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6801","nonce":132894}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6800","nonce":132894}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6807","nonce":132894}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6806","nonce":132894}]},"public_addr":"172.21.15.119:6801/132894","cluster_addr":"172.21.15.119:6800/132894","heartbeat_back_addr":"172.21.15.119:6807/132894","heartbeat_front_addr":"172.21.15.119:6806/132894","state":["exists","up"]},{"osd":3,"uuid":"351890ba-3634-423c-8e7b-7b84ac1438db","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":159,"down_at":175,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6802","nonce":132923}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6803","nonce":132923}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6808","nonce":132923}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6809","nonce":132923}]},"public_addr":"172.21.15.119:6802/132923","cluster_addr":"172.21.15.119:6803/132923","heartbeat_back_addr":"172.21.15.119:6808/132923","heartbeat_front_addr":"172.21.15.119:6809/132923","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-04-30T19:14:44.444308+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2023-04-30T19:15:25.673717+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":1},{"begin":8,"length":3},{"begin":14,"length":1},{"begin":16,"length":2},{"begin":20,"length":1},{"begin":22,"length":4},{"begin":27,"length":1},{"begin":31,"length":2},{"begin":36,"length":3},{"begin":40,"length":1},{"begin":42,"length":2},{"begin":45,"length":1},{"begin":49,"length":1},{"begin":51,"length":2},{"begin":55,"length":1},{"begin":58,"length":2},{"begin":61,"length":2},{"begin":64,"length":1},{"begin":68,"length":5},{"begin":76,"length":1},{"begin":78,"length":4},{"begin":83,"length":1},{"begin":87,"length":1},{"begin":90,"length":1},{"begin":92,"length":2},{"begin":95,"length":6},{"begin":103,"length":2},{"begin":107,"length":1},{"begin":109,"length":1},{"begin":112,"length":2},{"begin":115,"length":2},{"begin":122,"length":2},{"begin":125,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-04-30T19:17:30.485 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.1 is failed for ~190s , 2023-04-30T19:17:30.485 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~146s Ignoring malparsed backtrace: 2023-04-30T19:18:24.523 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.1 is failed for ~244s , 2023-04-30T19:18:24.523 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~200s , 2023-04-30T19:18:24.722 DEBUG:teuthology.orchestra.run.smithi029:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json Ignoring malparsed backtrace: 2023-04-30T19:19:18.561 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~254s , 2023-04-30T19:19:18.976 DEBUG:teuthology.orchestra.run.smithi029:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-04-30T19:19:19.108+0000 7f41a1111700 -1 WARNING: the following dangerous and experimental features are enabled: crimson Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_teuthology_69caab8ec6f44c61c991ae693e8e7b8cd859a896/teuthology/run_tasks.py", line 192, in run_tasks , suppress = manager.__exit__(*exc_info) , File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__ Ignoring malparsed backtrace: Traceback (most recent call last): , File "/home/teuthworker/src/git.ceph.com_teuthology_69caab8ec6f44c61c991ae693e8e7b8cd859a896/teuthology/orchestra/daemon/state.py", line 139, in stop , run.wait([self.proc], timeout=timeout) Ignoring malparsed backtrace: 2023-04-30T20:11:28.982 INFO:tasks.ceph:Archiving logs... , 2023-04-30T20:11:28.983 DEBUG:teuthology.misc:Transferring archived files from smithi029:/var/log/ceph to /home/teuthworker/archive/matan-2023-04-30_16:45:03-crimson-rados-main-distro-crimson-smithi/7258776/remote/smithi029/log , 2023-04-30T20:11:28.984 DEBUG:teuthology.orchestra.run.smithi029:> sudo tar c -f - -C /var/log/ceph -- . Ignoring malparsed backtrace: , 2023-04-30T20:11:40.996 INFO:teuthology.task.internal:core is compressed, try accessing gzip file ... , 2023-04-30T20:11:41.962 INFO:teuthology.task.internal: core looks like: /tmp/tmplr4n6vub: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from 'ceph_test_rados --max-ops 4000 --objects 50 --max-in-flight 16 --size 4000000 -', real uid: 1000, effective uid: 1000, real gid: 1268, effective gid: 1268, execfn: '/usr/bin/ceph_test_rados', platform: 'x86_64' Ignoring malparsed backtrace: ceph version 18.0.0-3633-g0ae30b6b (0ae30b6b0a0fdd9fa3bc438c826208438ac27aa2) reef (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7f210aeeeed4] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x1b0) [0x55d571d6c0e8] Ignoring malparsed backtrace: 2023-04-30T19:14:57.734 DEBUG:teuthology.orchestra.run.smithi029:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd reweight 0 1 , 2023-04-30T19:14:57.864+0000 7f44382f7700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-04-30T19:14:57.873+0000 7f44382f7700 -1 WARNING: the following dangerous and experimental features are enabled: crimson Ignoring malparsed backtrace: 2023-04-30T19:15:39.693+0000 7f3b58d61700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-04-30T19:15:39.903 INFO:teuthology.orchestra.run.smithi029.stdout: , 2023-04-30T19:15:39.903 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":176,"fsid":"ceaeeece-22e2-48ff-9295-38245c7f7da2","created":"2023-04-30T19:11:42.428777+0000","modified":"2023-04-30T19:15:26.679433+0000","last_up_change":"2023-04-30T19:15:25.673717+0000","last_in_change":"2023-04-30T19:11:44.667369+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:11:52.626169+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:11:53.007000+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.5,"raw_score_acting":1,"raw_score_stable":1,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:11:59.880342+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"161","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":142,"snap_epoch":161,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":2.2599999904632568,"score_stable":2.2599999904632568,"optimal_score":0.5,"raw_score_acting":1.1299999952316284,"raw_score_stable":1.1299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"e35756d0-679f-4288-82f8-700d9c05f7de","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":175,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6801","nonce":133225}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6800","nonce":133225}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6805","nonce":133225}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":133225}]},"public_addr":"172.21.15.29:6801/133225","cluster_addr":"172.21.15.29:6800/133225","heartbeat_back_addr":"172.21.15.29:6805/133225","heartbeat_front_addr":"172.21.15.29:6804/133225","state":["exists","up"]},{"osd":1,"uuid":"a63fbdcb-d6de-40c4-9079-ea506fa562a2","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":159,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6803","nonce":133252}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":133252}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":133252}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6807","nonce":133252}]},"public_addr":"172.21.15.29:6803/133252","cluster_addr":"172.21.15.29:6802/133252","heartbeat_back_addr":"172.21.15.29:6806/133252","heartbeat_front_addr":"172.21.15.29:6807/133252","state":["exists"]},{"osd":2,"uuid":"65457eae-1cda-486f-9c35-8f806493219f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":175,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6801","nonce":132894}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6800","nonce":132894}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6807","nonce":132894}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6806","nonce":132894}]},"public_addr":"172.21.15.119:6801/132894","cluster_addr":"172.21.15.119:6800/132894","heartbeat_back_addr":"172.21.15.119:6807/132894","heartbeat_front_addr":"172.21.15.119:6806/132894","state":["exists","up"]},{"osd":3,"uuid":"351890ba-3634-423c-8e7b-7b84ac1438db","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":159,"down_at":175,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6802","nonce":132923}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6803","nonce":132923}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6808","nonce":132923}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6809","nonce":132923}]},"public_addr":"172.21.15.119:6802/132923","cluster_addr":"172.21.15.119:6803/132923","heartbeat_back_addr":"172.21.15.119:6808/132923","heartbeat_front_addr":"172.21.15.119:6809/132923","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-04-30T19:14:44.444308+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2023-04-30T19:15:25.673717+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":1},{"begin":8,"length":3},{"begin":14,"length":1},{"begin":16,"length":2},{"begin":20,"length":1},{"begin":22,"length":4},{"begin":27,"length":1},{"begin":31,"length":2},{"begin":36,"length":3},{"begin":40,"length":1},{"begin":42,"length":2},{"begin":45,"length":1},{"begin":49,"length":1},{"begin":51,"length":2},{"begin":55,"length":1},{"begin":58,"length":2},{"begin":61,"length":2},{"begin":64,"length":1},{"begin":68,"length":5},{"begin":76,"length":1},{"begin":78,"length":4},{"begin":83,"length":1},{"begin":87,"length":1},{"begin":90,"length":1},{"begin":92,"length":2},{"begin":95,"length":6},{"begin":103,"length":2},{"begin":107,"length":1},{"begin":109,"length":1},{"begin":112,"length":2},{"begin":115,"length":2},{"begin":122,"length":2},{"begin":125,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} Ignoring malparsed backtrace: 2023-04-30T19:16:33.944 INFO:teuthology.orchestra.run.smithi029.stdout: , 2023-04-30T19:16:33.945 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":176,"fsid":"ceaeeece-22e2-48ff-9295-38245c7f7da2","created":"2023-04-30T19:11:42.428777+0000","modified":"2023-04-30T19:15:26.679433+0000","last_up_change":"2023-04-30T19:15:25.673717+0000","last_in_change":"2023-04-30T19:11:44.667369+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:11:52.626169+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:11:53.007000+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.5,"raw_score_acting":1,"raw_score_stable":1,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:11:59.880342+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"161","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":142,"snap_epoch":161,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":2.2599999904632568,"score_stable":2.2599999904632568,"optimal_score":0.5,"raw_score_acting":1.1299999952316284,"raw_score_stable":1.1299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"e35756d0-679f-4288-82f8-700d9c05f7de","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":175,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6801","nonce":133225}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6800","nonce":133225}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6805","nonce":133225}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":133225}]},"public_addr":"172.21.15.29:6801/133225","cluster_addr":"172.21.15.29:6800/133225","heartbeat_back_addr":"172.21.15.29:6805/133225","heartbeat_front_addr":"172.21.15.29:6804/133225","state":["exists","up"]},{"osd":1,"uuid":"a63fbdcb-d6de-40c4-9079-ea506fa562a2","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":159,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6803","nonce":133252}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":133252}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":133252}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6807","nonce":133252}]},"public_addr":"172.21.15.29:6803/133252","cluster_addr":"172.21.15.29:6802/133252","heartbeat_back_addr":"172.21.15.29:6806/133252","heartbeat_front_addr":"172.21.15.29:6807/133252","state":["exists"]},{"osd":2,"uuid":"65457eae-1cda-486f-9c35-8f806493219f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":175,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6801","nonce":132894}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6800","nonce":132894}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6807","nonce":132894}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6806","nonce":132894}]},"public_addr":"172.21.15.119:6801/132894","cluster_addr":"172.21.15.119:6800/132894","heartbeat_back_addr":"172.21.15.119:6807/132894","heartbeat_front_addr":"172.21.15.119:6806/132894","state":["exists","up"]},{"osd":3,"uuid":"351890ba-3634-423c-8e7b-7b84ac1438db","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":159,"down_at":175,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6802","nonce":132923}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6803","nonce":132923}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6808","nonce":132923}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6809","nonce":132923}]},"public_addr":"172.21.15.119:6802/132923","cluster_addr":"172.21.15.119:6803/132923","heartbeat_back_addr":"172.21.15.119:6808/132923","heartbeat_front_addr":"172.21.15.119:6809/132923","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-04-30T19:14:44.444308+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2023-04-30T19:15:25.673717+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":1},{"begin":8,"length":3},{"begin":14,"length":1},{"begin":16,"length":2},{"begin":20,"length":1},{"begin":22,"length":4},{"begin":27,"length":1},{"begin":31,"length":2},{"begin":36,"length":3},{"begin":40,"length":1},{"begin":42,"length":2},{"begin":45,"length":1},{"begin":49,"length":1},{"begin":51,"length":2},{"begin":55,"length":1},{"begin":58,"length":2},{"begin":61,"length":2},{"begin":64,"length":1},{"begin":68,"length":5},{"begin":76,"length":1},{"begin":78,"length":4},{"begin":83,"length":1},{"begin":87,"length":1},{"begin":90,"length":1},{"begin":92,"length":2},{"begin":95,"length":6},{"begin":103,"length":2},{"begin":107,"length":1},{"begin":109,"length":1},{"begin":112,"length":2},{"begin":115,"length":2},{"begin":122,"length":2},{"begin":125,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-04-30T19:16:36.447 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.1 is failed for ~136s Ignoring malparsed backtrace: 2023-04-30T19:17:27.943 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":176,"fsid":"ceaeeece-22e2-48ff-9295-38245c7f7da2","created":"2023-04-30T19:11:42.428777+0000","modified":"2023-04-30T19:15:26.679433+0000","last_up_change":"2023-04-30T19:15:25.673717+0000","last_in_change":"2023-04-30T19:11:44.667369+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:11:52.626169+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:11:53.007000+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.5,"raw_score_acting":1,"raw_score_stable":1,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:11:59.880342+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"161","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":142,"snap_epoch":161,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":2.2599999904632568,"score_stable":2.2599999904632568,"optimal_score":0.5,"raw_score_acting":1.1299999952316284,"raw_score_stable":1.1299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"e35756d0-679f-4288-82f8-700d9c05f7de","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":175,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6801","nonce":133225}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6800","nonce":133225}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6805","nonce":133225}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":133225}]},"public_addr":"172.21.15.29:6801/133225","cluster_addr":"172.21.15.29:6800/133225","heartbeat_back_addr":"172.21.15.29:6805/133225","heartbeat_front_addr":"172.21.15.29:6804/133225","state":["exists","up"]},{"osd":1,"uuid":"a63fbdcb-d6de-40c4-9079-ea506fa562a2","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":159,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6803","nonce":133252}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":133252}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":133252}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6807","nonce":133252}]},"public_addr":"172.21.15.29:6803/133252","cluster_addr":"172.21.15.29:6802/133252","heartbeat_back_addr":"172.21.15.29:6806/133252","heartbeat_front_addr":"172.21.15.29:6807/133252","state":["exists"]},{"osd":2,"uuid":"65457eae-1cda-486f-9c35-8f806493219f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":175,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6801","nonce":132894}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6800","nonce":132894}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6807","nonce":132894}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6806","nonce":132894}]},"public_addr":"172.21.15.119:6801/132894","cluster_addr":"172.21.15.119:6800/132894","heartbeat_back_addr":"172.21.15.119:6807/132894","heartbeat_front_addr":"172.21.15.119:6806/132894","state":["exists","up"]},{"osd":3,"uuid":"351890ba-3634-423c-8e7b-7b84ac1438db","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":159,"down_at":175,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6802","nonce":132923}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6803","nonce":132923}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6808","nonce":132923}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6809","nonce":132923}]},"public_addr":"172.21.15.119:6802/132923","cluster_addr":"172.21.15.119:6803/132923","heartbeat_back_addr":"172.21.15.119:6808/132923","heartbeat_front_addr":"172.21.15.119:6809/132923","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-04-30T19:14:44.444308+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2023-04-30T19:15:25.673717+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":1},{"begin":8,"length":3},{"begin":14,"length":1},{"begin":16,"length":2},{"begin":20,"length":1},{"begin":22,"length":4},{"begin":27,"length":1},{"begin":31,"length":2},{"begin":36,"length":3},{"begin":40,"length":1},{"begin":42,"length":2},{"begin":45,"length":1},{"begin":49,"length":1},{"begin":51,"length":2},{"begin":55,"length":1},{"begin":58,"length":2},{"begin":61,"length":2},{"begin":64,"length":1},{"begin":68,"length":5},{"begin":76,"length":1},{"begin":78,"length":4},{"begin":83,"length":1},{"begin":87,"length":1},{"begin":90,"length":1},{"begin":92,"length":2},{"begin":95,"length":6},{"begin":103,"length":2},{"begin":107,"length":1},{"begin":109,"length":1},{"begin":112,"length":2},{"begin":115,"length":2},{"begin":122,"length":2},{"begin":125,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-04-30T19:17:30.485 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.1 is failed for ~190s , 2023-04-30T19:17:30.485 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~146s Ignoring malparsed backtrace: 2023-04-30T19:18:24.523 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.1 is failed for ~244s , 2023-04-30T19:18:24.523 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~200s , 2023-04-30T19:18:24.722 DEBUG:teuthology.orchestra.run.smithi029:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json Ignoring malparsed backtrace: 2023-04-30T19:19:18.561 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~254s , 2023-04-30T19:19:18.976 DEBUG:teuthology.orchestra.run.smithi029:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-04-30T19:19:19.108+0000 7f41a1111700 -1 WARNING: the following dangerous and experimental features are enabled: crimson Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_teuthology_69caab8ec6f44c61c991ae693e8e7b8cd859a896/teuthology/run_tasks.py", line 192, in run_tasks , suppress = manager.__exit__(*exc_info) , File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__ Ignoring malparsed backtrace: Traceback (most recent call last): , File "/home/teuthworker/src/git.ceph.com_teuthology_69caab8ec6f44c61c991ae693e8e7b8cd859a896/teuthology/orchestra/daemon/state.py", line 139, in stop , run.wait([self.proc], timeout=timeout) Ignoring malparsed backtrace: 2023-04-30T20:11:28.982 INFO:tasks.ceph:Archiving logs... , 2023-04-30T20:11:28.983 DEBUG:teuthology.misc:Transferring archived files from smithi029:/var/log/ceph to /home/teuthworker/archive/matan-2023-04-30_16:45:03-crimson-rados-main-distro-crimson-smithi/7258776/remote/smithi029/log , 2023-04-30T20:11:28.984 DEBUG:teuthology.orchestra.run.smithi029:> sudo tar c -f - -C /var/log/ceph -- . Ignoring malparsed backtrace: , 2023-04-30T20:11:40.996 INFO:teuthology.task.internal:core is compressed, try accessing gzip file ... , 2023-04-30T20:11:41.962 INFO:teuthology.task.internal: core looks like: /tmp/tmplr4n6vub: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from 'ceph_test_rados --max-ops 4000 --objects 50 --max-in-flight 16 --size 4000000 -', real uid: 1000, effective uid: 1000, real gid: 1268, effective gid: 1268, execfn: '/usr/bin/ceph_test_rados', platform: 'x86_64' Ignoring malparsed backtrace: ceph version 18.0.0-3633-g0ae30b6b (0ae30b6b0a0fdd9fa3bc438c826208438ac27aa2) reef (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7f210aeeeed4] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x1b0) [0x55d571d6c0e8] Ignoring malparsed backtrace: 2023-04-30T19:14:57.734 DEBUG:teuthology.orchestra.run.smithi029:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd reweight 0 1 , 2023-04-30T19:14:57.864+0000 7f44382f7700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-04-30T19:14:57.873+0000 7f44382f7700 -1 WARNING: the following dangerous and experimental features are enabled: crimson Ignoring malparsed backtrace: 2023-04-30T19:15:39.693+0000 7f3b58d61700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-04-30T19:15:39.903 INFO:teuthology.orchestra.run.smithi029.stdout: , 2023-04-30T19:15:39.903 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":176,"fsid":"ceaeeece-22e2-48ff-9295-38245c7f7da2","created":"2023-04-30T19:11:42.428777+0000","modified":"2023-04-30T19:15:26.679433+0000","last_up_change":"2023-04-30T19:15:25.673717+0000","last_in_change":"2023-04-30T19:11:44.667369+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:11:52.626169+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:11:53.007000+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.5,"raw_score_acting":1,"raw_score_stable":1,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:11:59.880342+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"161","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":142,"snap_epoch":161,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":2.2599999904632568,"score_stable":2.2599999904632568,"optimal_score":0.5,"raw_score_acting":1.1299999952316284,"raw_score_stable":1.1299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"e35756d0-679f-4288-82f8-700d9c05f7de","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":175,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6801","nonce":133225}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6800","nonce":133225}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6805","nonce":133225}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":133225}]},"public_addr":"172.21.15.29:6801/133225","cluster_addr":"172.21.15.29:6800/133225","heartbeat_back_addr":"172.21.15.29:6805/133225","heartbeat_front_addr":"172.21.15.29:6804/133225","state":["exists","up"]},{"osd":1,"uuid":"a63fbdcb-d6de-40c4-9079-ea506fa562a2","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":159,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6803","nonce":133252}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":133252}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":133252}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6807","nonce":133252}]},"public_addr":"172.21.15.29:6803/133252","cluster_addr":"172.21.15.29:6802/133252","heartbeat_back_addr":"172.21.15.29:6806/133252","heartbeat_front_addr":"172.21.15.29:6807/133252","state":["exists"]},{"osd":2,"uuid":"65457eae-1cda-486f-9c35-8f806493219f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":175,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6801","nonce":132894}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6800","nonce":132894}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6807","nonce":132894}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6806","nonce":132894}]},"public_addr":"172.21.15.119:6801/132894","cluster_addr":"172.21.15.119:6800/132894","heartbeat_back_addr":"172.21.15.119:6807/132894","heartbeat_front_addr":"172.21.15.119:6806/132894","state":["exists","up"]},{"osd":3,"uuid":"351890ba-3634-423c-8e7b-7b84ac1438db","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":159,"down_at":175,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6802","nonce":132923}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6803","nonce":132923}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6808","nonce":132923}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6809","nonce":132923}]},"public_addr":"172.21.15.119:6802/132923","cluster_addr":"172.21.15.119:6803/132923","heartbeat_back_addr":"172.21.15.119:6808/132923","heartbeat_front_addr":"172.21.15.119:6809/132923","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-04-30T19:14:44.444308+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2023-04-30T19:15:25.673717+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":1},{"begin":8,"length":3},{"begin":14,"length":1},{"begin":16,"length":2},{"begin":20,"length":1},{"begin":22,"length":4},{"begin":27,"length":1},{"begin":31,"length":2},{"begin":36,"length":3},{"begin":40,"length":1},{"begin":42,"length":2},{"begin":45,"length":1},{"begin":49,"length":1},{"begin":51,"length":2},{"begin":55,"length":1},{"begin":58,"length":2},{"begin":61,"length":2},{"begin":64,"length":1},{"begin":68,"length":5},{"begin":76,"length":1},{"begin":78,"length":4},{"begin":83,"length":1},{"begin":87,"length":1},{"begin":90,"length":1},{"begin":92,"length":2},{"begin":95,"length":6},{"begin":103,"length":2},{"begin":107,"length":1},{"begin":109,"length":1},{"begin":112,"length":2},{"begin":115,"length":2},{"begin":122,"length":2},{"begin":125,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} Ignoring malparsed backtrace: 2023-04-30T19:16:33.944 INFO:teuthology.orchestra.run.smithi029.stdout: , 2023-04-30T19:16:33.945 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":176,"fsid":"ceaeeece-22e2-48ff-9295-38245c7f7da2","created":"2023-04-30T19:11:42.428777+0000","modified":"2023-04-30T19:15:26.679433+0000","last_up_change":"2023-04-30T19:15:25.673717+0000","last_in_change":"2023-04-30T19:11:44.667369+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:11:52.626169+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:11:53.007000+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.5,"raw_score_acting":1,"raw_score_stable":1,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:11:59.880342+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"161","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":142,"snap_epoch":161,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":2.2599999904632568,"score_stable":2.2599999904632568,"optimal_score":0.5,"raw_score_acting":1.1299999952316284,"raw_score_stable":1.1299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"e35756d0-679f-4288-82f8-700d9c05f7de","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":175,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6801","nonce":133225}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6800","nonce":133225}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6805","nonce":133225}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":133225}]},"public_addr":"172.21.15.29:6801/133225","cluster_addr":"172.21.15.29:6800/133225","heartbeat_back_addr":"172.21.15.29:6805/133225","heartbeat_front_addr":"172.21.15.29:6804/133225","state":["exists","up"]},{"osd":1,"uuid":"a63fbdcb-d6de-40c4-9079-ea506fa562a2","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":159,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6803","nonce":133252}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":133252}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":133252}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6807","nonce":133252}]},"public_addr":"172.21.15.29:6803/133252","cluster_addr":"172.21.15.29:6802/133252","heartbeat_back_addr":"172.21.15.29:6806/133252","heartbeat_front_addr":"172.21.15.29:6807/133252","state":["exists"]},{"osd":2,"uuid":"65457eae-1cda-486f-9c35-8f806493219f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":175,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6801","nonce":132894}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6800","nonce":132894}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6807","nonce":132894}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6806","nonce":132894}]},"public_addr":"172.21.15.119:6801/132894","cluster_addr":"172.21.15.119:6800/132894","heartbeat_back_addr":"172.21.15.119:6807/132894","heartbeat_front_addr":"172.21.15.119:6806/132894","state":["exists","up"]},{"osd":3,"uuid":"351890ba-3634-423c-8e7b-7b84ac1438db","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":159,"down_at":175,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6802","nonce":132923}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6803","nonce":132923}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6808","nonce":132923}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6809","nonce":132923}]},"public_addr":"172.21.15.119:6802/132923","cluster_addr":"172.21.15.119:6803/132923","heartbeat_back_addr":"172.21.15.119:6808/132923","heartbeat_front_addr":"172.21.15.119:6809/132923","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-04-30T19:14:44.444308+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2023-04-30T19:15:25.673717+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":1},{"begin":8,"length":3},{"begin":14,"length":1},{"begin":16,"length":2},{"begin":20,"length":1},{"begin":22,"length":4},{"begin":27,"length":1},{"begin":31,"length":2},{"begin":36,"length":3},{"begin":40,"length":1},{"begin":42,"length":2},{"begin":45,"length":1},{"begin":49,"length":1},{"begin":51,"length":2},{"begin":55,"length":1},{"begin":58,"length":2},{"begin":61,"length":2},{"begin":64,"length":1},{"begin":68,"length":5},{"begin":76,"length":1},{"begin":78,"length":4},{"begin":83,"length":1},{"begin":87,"length":1},{"begin":90,"length":1},{"begin":92,"length":2},{"begin":95,"length":6},{"begin":103,"length":2},{"begin":107,"length":1},{"begin":109,"length":1},{"begin":112,"length":2},{"begin":115,"length":2},{"begin":122,"length":2},{"begin":125,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-04-30T19:16:36.447 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.1 is failed for ~136s Ignoring malparsed backtrace: 2023-04-30T19:17:27.943 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":176,"fsid":"ceaeeece-22e2-48ff-9295-38245c7f7da2","created":"2023-04-30T19:11:42.428777+0000","modified":"2023-04-30T19:15:26.679433+0000","last_up_change":"2023-04-30T19:15:25.673717+0000","last_in_change":"2023-04-30T19:11:44.667369+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:11:52.626169+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:11:53.007000+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.5,"raw_score_acting":1,"raw_score_stable":1,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:11:59.880342+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"161","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":142,"snap_epoch":161,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":2.2599999904632568,"score_stable":2.2599999904632568,"optimal_score":0.5,"raw_score_acting":1.1299999952316284,"raw_score_stable":1.1299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"e35756d0-679f-4288-82f8-700d9c05f7de","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":175,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6801","nonce":133225}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6800","nonce":133225}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6805","nonce":133225}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":133225}]},"public_addr":"172.21.15.29:6801/133225","cluster_addr":"172.21.15.29:6800/133225","heartbeat_back_addr":"172.21.15.29:6805/133225","heartbeat_front_addr":"172.21.15.29:6804/133225","state":["exists","up"]},{"osd":1,"uuid":"a63fbdcb-d6de-40c4-9079-ea506fa562a2","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":159,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6803","nonce":133252}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":133252}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":133252}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6807","nonce":133252}]},"public_addr":"172.21.15.29:6803/133252","cluster_addr":"172.21.15.29:6802/133252","heartbeat_back_addr":"172.21.15.29:6806/133252","heartbeat_front_addr":"172.21.15.29:6807/133252","state":["exists"]},{"osd":2,"uuid":"65457eae-1cda-486f-9c35-8f806493219f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":175,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6801","nonce":132894}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6800","nonce":132894}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6807","nonce":132894}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6806","nonce":132894}]},"public_addr":"172.21.15.119:6801/132894","cluster_addr":"172.21.15.119:6800/132894","heartbeat_back_addr":"172.21.15.119:6807/132894","heartbeat_front_addr":"172.21.15.119:6806/132894","state":["exists","up"]},{"osd":3,"uuid":"351890ba-3634-423c-8e7b-7b84ac1438db","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":159,"down_at":175,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6802","nonce":132923}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6803","nonce":132923}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6808","nonce":132923}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6809","nonce":132923}]},"public_addr":"172.21.15.119:6802/132923","cluster_addr":"172.21.15.119:6803/132923","heartbeat_back_addr":"172.21.15.119:6808/132923","heartbeat_front_addr":"172.21.15.119:6809/132923","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-04-30T19:14:44.444308+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2023-04-30T19:15:25.673717+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":1},{"begin":8,"length":3},{"begin":14,"length":1},{"begin":16,"length":2},{"begin":20,"length":1},{"begin":22,"length":4},{"begin":27,"length":1},{"begin":31,"length":2},{"begin":36,"length":3},{"begin":40,"length":1},{"begin":42,"length":2},{"begin":45,"length":1},{"begin":49,"length":1},{"begin":51,"length":2},{"begin":55,"length":1},{"begin":58,"length":2},{"begin":61,"length":2},{"begin":64,"length":1},{"begin":68,"length":5},{"begin":76,"length":1},{"begin":78,"length":4},{"begin":83,"length":1},{"begin":87,"length":1},{"begin":90,"length":1},{"begin":92,"length":2},{"begin":95,"length":6},{"begin":103,"length":2},{"begin":107,"length":1},{"begin":109,"length":1},{"begin":112,"length":2},{"begin":115,"length":2},{"begin":122,"length":2},{"begin":125,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-04-30T19:17:30.485 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.1 is failed for ~190s , 2023-04-30T19:17:30.485 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~146s Ignoring malparsed backtrace: 2023-04-30T19:18:24.523 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.1 is failed for ~244s , 2023-04-30T19:18:24.523 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~200s , 2023-04-30T19:18:24.722 DEBUG:teuthology.orchestra.run.smithi029:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json Ignoring malparsed backtrace: 2023-04-30T19:19:18.561 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~254s , 2023-04-30T19:19:18.976 DEBUG:teuthology.orchestra.run.smithi029:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-04-30T19:19:19.108+0000 7f41a1111700 -1 WARNING: the following dangerous and experimental features are enabled: crimson Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_teuthology_69caab8ec6f44c61c991ae693e8e7b8cd859a896/teuthology/run_tasks.py", line 192, in run_tasks , suppress = manager.__exit__(*exc_info) , File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__ Ignoring malparsed backtrace: Traceback (most recent call last): , File "/home/teuthworker/src/git.ceph.com_teuthology_69caab8ec6f44c61c991ae693e8e7b8cd859a896/teuthology/orchestra/daemon/state.py", line 139, in stop , run.wait([self.proc], timeout=timeout) Ignoring malparsed backtrace: 2023-04-30T20:11:28.982 INFO:tasks.ceph:Archiving logs... , 2023-04-30T20:11:28.983 DEBUG:teuthology.misc:Transferring archived files from smithi029:/var/log/ceph to /home/teuthworker/archive/matan-2023-04-30_16:45:03-crimson-rados-main-distro-crimson-smithi/7258776/remote/smithi029/log , 2023-04-30T20:11:28.984 DEBUG:teuthology.orchestra.run.smithi029:> sudo tar c -f - -C /var/log/ceph -- . Ignoring malparsed backtrace: , 2023-04-30T20:11:40.996 INFO:teuthology.task.internal:core is compressed, try accessing gzip file ... , 2023-04-30T20:11:41.962 INFO:teuthology.task.internal: core looks like: /tmp/tmplr4n6vub: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from 'ceph_test_rados --max-ops 4000 --objects 50 --max-in-flight 16 --size 4000000 -', real uid: 1000, effective uid: 1000, real gid: 1268, effective gid: 1268, execfn: '/usr/bin/ceph_test_rados', platform: 'x86_64' Ignoring malparsed backtrace: ceph version 18.0.0-3633-g0ae30b6b (0ae30b6b0a0fdd9fa3bc438c826208438ac27aa2) reef (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7f210aeeeed4] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x1b0) [0x55d571d6c0e8] Ignoring malparsed backtrace: 2023-04-30T19:14:57.734 DEBUG:teuthology.orchestra.run.smithi029:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd reweight 0 1 , 2023-04-30T19:14:57.864+0000 7f44382f7700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-04-30T19:14:57.873+0000 7f44382f7700 -1 WARNING: the following dangerous and experimental features are enabled: crimson Ignoring malparsed backtrace: 2023-04-30T19:15:39.693+0000 7f3b58d61700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-04-30T19:15:39.903 INFO:teuthology.orchestra.run.smithi029.stdout: , 2023-04-30T19:15:39.903 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":176,"fsid":"ceaeeece-22e2-48ff-9295-38245c7f7da2","created":"2023-04-30T19:11:42.428777+0000","modified":"2023-04-30T19:15:26.679433+0000","last_up_change":"2023-04-30T19:15:25.673717+0000","last_in_change":"2023-04-30T19:11:44.667369+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:11:52.626169+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:11:53.007000+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.5,"raw_score_acting":1,"raw_score_stable":1,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:11:59.880342+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"161","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":142,"snap_epoch":161,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":2.2599999904632568,"score_stable":2.2599999904632568,"optimal_score":0.5,"raw_score_acting":1.1299999952316284,"raw_score_stable":1.1299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"e35756d0-679f-4288-82f8-700d9c05f7de","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":175,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6801","nonce":133225}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6800","nonce":133225}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6805","nonce":133225}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":133225}]},"public_addr":"172.21.15.29:6801/133225","cluster_addr":"172.21.15.29:6800/133225","heartbeat_back_addr":"172.21.15.29:6805/133225","heartbeat_front_addr":"172.21.15.29:6804/133225","state":["exists","up"]},{"osd":1,"uuid":"a63fbdcb-d6de-40c4-9079-ea506fa562a2","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":159,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6803","nonce":133252}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":133252}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":133252}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6807","nonce":133252}]},"public_addr":"172.21.15.29:6803/133252","cluster_addr":"172.21.15.29:6802/133252","heartbeat_back_addr":"172.21.15.29:6806/133252","heartbeat_front_addr":"172.21.15.29:6807/133252","state":["exists"]},{"osd":2,"uuid":"65457eae-1cda-486f-9c35-8f806493219f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":175,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6801","nonce":132894}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6800","nonce":132894}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6807","nonce":132894}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6806","nonce":132894}]},"public_addr":"172.21.15.119:6801/132894","cluster_addr":"172.21.15.119:6800/132894","heartbeat_back_addr":"172.21.15.119:6807/132894","heartbeat_front_addr":"172.21.15.119:6806/132894","state":["exists","up"]},{"osd":3,"uuid":"351890ba-3634-423c-8e7b-7b84ac1438db","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":159,"down_at":175,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6802","nonce":132923}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6803","nonce":132923}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6808","nonce":132923}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6809","nonce":132923}]},"public_addr":"172.21.15.119:6802/132923","cluster_addr":"172.21.15.119:6803/132923","heartbeat_back_addr":"172.21.15.119:6808/132923","heartbeat_front_addr":"172.21.15.119:6809/132923","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-04-30T19:14:44.444308+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2023-04-30T19:15:25.673717+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":1},{"begin":8,"length":3},{"begin":14,"length":1},{"begin":16,"length":2},{"begin":20,"length":1},{"begin":22,"length":4},{"begin":27,"length":1},{"begin":31,"length":2},{"begin":36,"length":3},{"begin":40,"length":1},{"begin":42,"length":2},{"begin":45,"length":1},{"begin":49,"length":1},{"begin":51,"length":2},{"begin":55,"length":1},{"begin":58,"length":2},{"begin":61,"length":2},{"begin":64,"length":1},{"begin":68,"length":5},{"begin":76,"length":1},{"begin":78,"length":4},{"begin":83,"length":1},{"begin":87,"length":1},{"begin":90,"length":1},{"begin":92,"length":2},{"begin":95,"length":6},{"begin":103,"length":2},{"begin":107,"length":1},{"begin":109,"length":1},{"begin":112,"length":2},{"begin":115,"length":2},{"begin":122,"length":2},{"begin":125,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} Ignoring malparsed backtrace: 2023-04-30T19:16:33.944 INFO:teuthology.orchestra.run.smithi029.stdout: , 2023-04-30T19:16:33.945 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":176,"fsid":"ceaeeece-22e2-48ff-9295-38245c7f7da2","created":"2023-04-30T19:11:42.428777+0000","modified":"2023-04-30T19:15:26.679433+0000","last_up_change":"2023-04-30T19:15:25.673717+0000","last_in_change":"2023-04-30T19:11:44.667369+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:11:52.626169+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:11:53.007000+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.5,"raw_score_acting":1,"raw_score_stable":1,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:11:59.880342+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"161","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":142,"snap_epoch":161,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":2.2599999904632568,"score_stable":2.2599999904632568,"optimal_score":0.5,"raw_score_acting":1.1299999952316284,"raw_score_stable":1.1299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"e35756d0-679f-4288-82f8-700d9c05f7de","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":175,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6801","nonce":133225}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6800","nonce":133225}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6805","nonce":133225}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":133225}]},"public_addr":"172.21.15.29:6801/133225","cluster_addr":"172.21.15.29:6800/133225","heartbeat_back_addr":"172.21.15.29:6805/133225","heartbeat_front_addr":"172.21.15.29:6804/133225","state":["exists","up"]},{"osd":1,"uuid":"a63fbdcb-d6de-40c4-9079-ea506fa562a2","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":159,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6803","nonce":133252}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":133252}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":133252}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6807","nonce":133252}]},"public_addr":"172.21.15.29:6803/133252","cluster_addr":"172.21.15.29:6802/133252","heartbeat_back_addr":"172.21.15.29:6806/133252","heartbeat_front_addr":"172.21.15.29:6807/133252","state":["exists"]},{"osd":2,"uuid":"65457eae-1cda-486f-9c35-8f806493219f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":175,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6801","nonce":132894}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6800","nonce":132894}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6807","nonce":132894}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6806","nonce":132894}]},"public_addr":"172.21.15.119:6801/132894","cluster_addr":"172.21.15.119:6800/132894","heartbeat_back_addr":"172.21.15.119:6807/132894","heartbeat_front_addr":"172.21.15.119:6806/132894","state":["exists","up"]},{"osd":3,"uuid":"351890ba-3634-423c-8e7b-7b84ac1438db","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":159,"down_at":175,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6802","nonce":132923}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6803","nonce":132923}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6808","nonce":132923}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6809","nonce":132923}]},"public_addr":"172.21.15.119:6802/132923","cluster_addr":"172.21.15.119:6803/132923","heartbeat_back_addr":"172.21.15.119:6808/132923","heartbeat_front_addr":"172.21.15.119:6809/132923","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-04-30T19:14:44.444308+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2023-04-30T19:15:25.673717+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":1},{"begin":8,"length":3},{"begin":14,"length":1},{"begin":16,"length":2},{"begin":20,"length":1},{"begin":22,"length":4},{"begin":27,"length":1},{"begin":31,"length":2},{"begin":36,"length":3},{"begin":40,"length":1},{"begin":42,"length":2},{"begin":45,"length":1},{"begin":49,"length":1},{"begin":51,"length":2},{"begin":55,"length":1},{"begin":58,"length":2},{"begin":61,"length":2},{"begin":64,"length":1},{"begin":68,"length":5},{"begin":76,"length":1},{"begin":78,"length":4},{"begin":83,"length":1},{"begin":87,"length":1},{"begin":90,"length":1},{"begin":92,"length":2},{"begin":95,"length":6},{"begin":103,"length":2},{"begin":107,"length":1},{"begin":109,"length":1},{"begin":112,"length":2},{"begin":115,"length":2},{"begin":122,"length":2},{"begin":125,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-04-30T19:16:36.447 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.1 is failed for ~136s Ignoring malparsed backtrace: 2023-04-30T19:17:27.943 INFO:teuthology.orchestra.run.smithi029.stdout:{"epoch":176,"fsid":"ceaeeece-22e2-48ff-9295-38245c7f7da2","created":"2023-04-30T19:11:42.428777+0000","modified":"2023-04-30T19:15:26.679433+0000","last_up_change":"2023-04-30T19:15:25.673717+0000","last_in_change":"2023-04-30T19:11:44.667369+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:11:52.626169+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:11:53.007000+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.5,"raw_score_acting":1,"raw_score_stable":1,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:11:59.880342+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"161","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":142,"snap_epoch":161,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":2.2599999904632568,"score_stable":2.2599999904632568,"optimal_score":0.5,"raw_score_acting":1.1299999952316284,"raw_score_stable":1.1299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"e35756d0-679f-4288-82f8-700d9c05f7de","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":175,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6801","nonce":133225}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6800","nonce":133225}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6805","nonce":133225}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6804","nonce":133225}]},"public_addr":"172.21.15.29:6801/133225","cluster_addr":"172.21.15.29:6800/133225","heartbeat_back_addr":"172.21.15.29:6805/133225","heartbeat_front_addr":"172.21.15.29:6804/133225","state":["exists","up"]},{"osd":1,"uuid":"a63fbdcb-d6de-40c4-9079-ea506fa562a2","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":159,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6803","nonce":133252}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6802","nonce":133252}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6806","nonce":133252}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.29:6807","nonce":133252}]},"public_addr":"172.21.15.29:6803/133252","cluster_addr":"172.21.15.29:6802/133252","heartbeat_back_addr":"172.21.15.29:6806/133252","heartbeat_front_addr":"172.21.15.29:6807/133252","state":["exists"]},{"osd":2,"uuid":"65457eae-1cda-486f-9c35-8f806493219f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":175,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6801","nonce":132894}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6800","nonce":132894}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6807","nonce":132894}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6806","nonce":132894}]},"public_addr":"172.21.15.119:6801/132894","cluster_addr":"172.21.15.119:6800/132894","heartbeat_back_addr":"172.21.15.119:6807/132894","heartbeat_front_addr":"172.21.15.119:6806/132894","state":["exists","up"]},{"osd":3,"uuid":"351890ba-3634-423c-8e7b-7b84ac1438db","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":159,"down_at":175,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6802","nonce":132923}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6803","nonce":132923}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6808","nonce":132923}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.119:6809","nonce":132923}]},"public_addr":"172.21.15.119:6802/132923","cluster_addr":"172.21.15.119:6803/132923","heartbeat_back_addr":"172.21.15.119:6808/132923","heartbeat_front_addr":"172.21.15.119:6809/132923","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-04-30T19:14:44.444308+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2023-04-30T19:15:25.673717+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":1},{"begin":8,"length":3},{"begin":14,"length":1},{"begin":16,"length":2},{"begin":20,"length":1},{"begin":22,"length":4},{"begin":27,"length":1},{"begin":31,"length":2},{"begin":36,"length":3},{"begin":40,"length":1},{"begin":42,"length":2},{"begin":45,"length":1},{"begin":49,"length":1},{"begin":51,"length":2},{"begin":55,"length":1},{"begin":58,"length":2},{"begin":61,"length":2},{"begin":64,"length":1},{"begin":68,"length":5},{"begin":76,"length":1},{"begin":78,"length":4},{"begin":83,"length":1},{"begin":87,"length":1},{"begin":90,"length":1},{"begin":92,"length":2},{"begin":95,"length":6},{"begin":103,"length":2},{"begin":107,"length":1},{"begin":109,"length":1},{"begin":112,"length":2},{"begin":115,"length":2},{"begin":122,"length":2},{"begin":125,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-04-30T19:17:30.485 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.1 is failed for ~190s , 2023-04-30T19:17:30.485 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~146s Ignoring malparsed backtrace: 2023-04-30T19:18:24.523 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.1 is failed for ~244s , 2023-04-30T19:18:24.523 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~200s , 2023-04-30T19:18:24.722 DEBUG:teuthology.orchestra.run.smithi029:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json Ignoring malparsed backtrace: 2023-04-30T19:19:18.561 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~254s , 2023-04-30T19:19:18.976 DEBUG:teuthology.orchestra.run.smithi029:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-04-30T19:19:19.108+0000 7f41a1111700 -1 WARNING: the following dangerous and experimental features are enabled: crimson Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_teuthology_69caab8ec6f44c61c991ae693e8e7b8cd859a896/teuthology/run_tasks.py", line 192, in run_tasks , suppress = manager.__exit__(*exc_info) , File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__ Ignoring malparsed backtrace: Traceback (most recent call last): , File "/home/teuthworker/src/git.ceph.com_teuthology_69caab8ec6f44c61c991ae693e8e7b8cd859a896/teuthology/orchestra/daemon/state.py", line 139, in stop , run.wait([self.proc], timeout=timeout) Ignoring malparsed backtrace: 2023-04-30T20:11:28.982 INFO:tasks.ceph:Archiving logs... , 2023-04-30T20:11:28.983 DEBUG:teuthology.misc:Transferring archived files from smithi029:/var/log/ceph to /home/teuthworker/archive/matan-2023-04-30_16:45:03-crimson-rados-main-distro-crimson-smithi/7258776/remote/smithi029/log , 2023-04-30T20:11:28.984 DEBUG:teuthology.orchestra.run.smithi029:> sudo tar c -f - -C /var/log/ceph -- . Ignoring malparsed backtrace: , 2023-04-30T20:11:40.996 INFO:teuthology.task.internal:core is compressed, try accessing gzip file ... , 2023-04-30T20:11:41.962 INFO:teuthology.task.internal: core looks like: /tmp/tmplr4n6vub: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from 'ceph_test_rados --max-ops 4000 --objects 50 --max-in-flight 16 --size 4000000 -', real uid: 1000, effective uid: 1000, real gid: 1268, effective gid: 1268, execfn: '/usr/bin/ceph_test_rados', platform: 'x86_64' Ignoring malparsed backtrace: ceph version 18.0.0-3633-g0ae30b6b (0ae30b6b0a0fdd9fa3bc438c826208438ac27aa2) reef (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7fb1daf3ced4] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x1b0) [0x56211f0270e8] Ignoring malparsed backtrace: 2023-04-30T19:08:26.050 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~33s , 2023-04-30T19:08:27.301 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-04-30T19:08:27.433+0000 7fc7b946d700 -1 WARNING: the following dangerous and experimental features are enabled: crimson Ignoring malparsed backtrace: 2023-04-30T19:09:18.082+0000 7f72e4d0f700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-04-30T19:09:18.289 INFO:teuthology.orchestra.run.smithi032.stdout: , 2023-04-30T19:09:18.289 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":174,"fsid":"4e6dbd7f-aa02-42c5-aed2-840d605d34fe","created":"2023-04-30T19:05:14.879777+0000","modified":"2023-04-30T19:08:47.898007+0000","last_up_change":"2023-04-30T19:08:46.896611+0000","last_in_change":"2023-04-30T19:05:17.167602+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:05:23.030847+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.25,"raw_score_acting":1,"raw_score_stable":1,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:05:25.519290+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2.5,"score_stable":2.5,"optimal_score":0.5,"raw_score_acting":1.25,"raw_score_stable":1.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:05:32.499487+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"157","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":139,"snap_epoch":157,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":2.2599999904632568,"score_stable":2.2599999904632568,"optimal_score":0.5,"raw_score_acting":1.1299999952316284,"raw_score_stable":1.1299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"d6178da2-c707-4fdf-abd5-248d8c097ac6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":173,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6801","nonce":133248}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6800","nonce":133248}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":133248}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6805","nonce":133248}]},"public_addr":"172.21.15.32:6801/133248","cluster_addr":"172.21.15.32:6800/133248","heartbeat_back_addr":"172.21.15.32:6804/133248","heartbeat_front_addr":"172.21.15.32:6805/133248","state":["exists","up"]},{"osd":1,"uuid":"24cda7e5-680d-43f3-9944-7bc94b5a5495","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":173,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":133275}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6803","nonce":133275}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6807","nonce":133275}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":133275}]},"public_addr":"172.21.15.32:6802/133275","cluster_addr":"172.21.15.32:6803/133275","heartbeat_back_addr":"172.21.15.32:6807/133275","heartbeat_front_addr":"172.21.15.32:6806/133275","state":["exists","up"]},{"osd":2,"uuid":"e23153ff-eee1-4ae8-8936-737004527966","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":163,"down_at":173,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":132851}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":132851}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":132851}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":132851}]},"public_addr":"172.21.15.144:6802/132851","cluster_addr":"172.21.15.144:6803/132851","heartbeat_back_addr":"172.21.15.144:6807/132851","heartbeat_front_addr":"172.21.15.144:6806/132851","state":["exists"]},{"osd":3,"uuid":"3c8a1873-5a52-435a-aee4-f7c5611dcf21","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":16,"down_at":163,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":132879}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":132879}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6809","nonce":132879}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6808","nonce":132879}]},"public_addr":"172.21.15.144:6804/132879","cluster_addr":"172.21.15.144:6805/132879","heartbeat_back_addr":"172.21.15.144:6809/132879","heartbeat_front_addr":"172.21.15.144:6808/132879","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"2023-04-30T19:08:46.896611+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2023-04-30T19:08:10.886845+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":1},{"begin":8,"length":3},{"begin":14,"length":1},{"begin":16,"length":2},{"begin":20,"length":1},{"begin":22,"length":4},{"begin":27,"length":1},{"begin":31,"length":2},{"begin":36,"length":3},{"begin":40,"length":1},{"begin":42,"length":2},{"begin":45,"length":1},{"begin":49,"length":1},{"begin":51,"length":2},{"begin":56,"length":1},{"begin":58,"length":2},{"begin":61,"length":1},{"begin":63,"length":1},{"begin":65,"length":5},{"begin":73,"length":1},{"begin":75,"length":4},{"begin":80,"length":2},{"begin":84,"length":1},{"begin":87,"length":1},{"begin":89,"length":2},{"begin":92,"length":2},{"begin":95,"length":3},{"begin":99,"length":3},{"begin":106,"length":1},{"begin":109,"length":2},{"begin":112,"length":1},{"begin":114,"length":1},{"begin":116,"length":2},{"begin":120,"length":1},{"begin":122,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} Ignoring malparsed backtrace: 2023-04-30T19:10:05.106+0000 7fecee6e6700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-04-30T19:10:05.310 INFO:teuthology.orchestra.run.smithi032.stdout: , 2023-04-30T19:10:05.311 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":174,"fsid":"4e6dbd7f-aa02-42c5-aed2-840d605d34fe","created":"2023-04-30T19:05:14.879777+0000","modified":"2023-04-30T19:08:47.898007+0000","last_up_change":"2023-04-30T19:08:46.896611+0000","last_in_change":"2023-04-30T19:05:17.167602+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:05:23.030847+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.25,"raw_score_acting":1,"raw_score_stable":1,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:05:25.519290+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2.5,"score_stable":2.5,"optimal_score":0.5,"raw_score_acting":1.25,"raw_score_stable":1.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:05:32.499487+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"157","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":139,"snap_epoch":157,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":2.2599999904632568,"score_stable":2.2599999904632568,"optimal_score":0.5,"raw_score_acting":1.1299999952316284,"raw_score_stable":1.1299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"d6178da2-c707-4fdf-abd5-248d8c097ac6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":173,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6801","nonce":133248}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6800","nonce":133248}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":133248}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6805","nonce":133248}]},"public_addr":"172.21.15.32:6801/133248","cluster_addr":"172.21.15.32:6800/133248","heartbeat_back_addr":"172.21.15.32:6804/133248","heartbeat_front_addr":"172.21.15.32:6805/133248","state":["exists","up"]},{"osd":1,"uuid":"24cda7e5-680d-43f3-9944-7bc94b5a5495","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":173,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":133275}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6803","nonce":133275}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6807","nonce":133275}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":133275}]},"public_addr":"172.21.15.32:6802/133275","cluster_addr":"172.21.15.32:6803/133275","heartbeat_back_addr":"172.21.15.32:6807/133275","heartbeat_front_addr":"172.21.15.32:6806/133275","state":["exists","up"]},{"osd":2,"uuid":"e23153ff-eee1-4ae8-8936-737004527966","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":163,"down_at":173,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":132851}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":132851}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":132851}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":132851}]},"public_addr":"172.21.15.144:6802/132851","cluster_addr":"172.21.15.144:6803/132851","heartbeat_back_addr":"172.21.15.144:6807/132851","heartbeat_front_addr":"172.21.15.144:6806/132851","state":["exists"]},{"osd":3,"uuid":"3c8a1873-5a52-435a-aee4-f7c5611dcf21","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":16,"down_at":163,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":132879}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":132879}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6809","nonce":132879}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6808","nonce":132879}]},"public_addr":"172.21.15.144:6804/132879","cluster_addr":"172.21.15.144:6805/132879","heartbeat_back_addr":"172.21.15.144:6809/132879","heartbeat_front_addr":"172.21.15.144:6808/132879","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"2023-04-30T19:08:46.896611+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2023-04-30T19:08:10.886845+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":1},{"begin":8,"length":3},{"begin":14,"length":1},{"begin":16,"length":2},{"begin":20,"length":1},{"begin":22,"length":4},{"begin":27,"length":1},{"begin":31,"length":2},{"begin":36,"length":3},{"begin":40,"length":1},{"begin":42,"length":2},{"begin":45,"length":1},{"begin":49,"length":1},{"begin":51,"length":2},{"begin":56,"length":1},{"begin":58,"length":2},{"begin":61,"length":1},{"begin":63,"length":1},{"begin":65,"length":5},{"begin":73,"length":1},{"begin":75,"length":4},{"begin":80,"length":2},{"begin":84,"length":1},{"begin":87,"length":1},{"begin":89,"length":2},{"begin":92,"length":2},{"begin":95,"length":3},{"begin":99,"length":3},{"begin":106,"length":1},{"begin":109,"length":2},{"begin":112,"length":1},{"begin":114,"length":1},{"begin":116,"length":2},{"begin":120,"length":1},{"begin":122,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} Ignoring malparsed backtrace: 2023-04-30T19:10:49.638 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.2 is failed for ~149s , 2023-04-30T19:10:49.638 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~177s , 2023-04-30T19:10:49.911+0000 7fda1187e700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.1 v2:172.21.15.32:6802/133275 for 29sec e174 v174) Ignoring malparsed backtrace: 2023-04-30T19:11:36.582 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-04-30T19:11:36.712+0000 7f03aa294700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-04-30T19:11:36.719+0000 7f03aa294700 -1 WARNING: the following dangerous and experimental features are enabled: crimson Ignoring malparsed backtrace: 2023-04-30T19:12:20.291 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-04-30T19:12:20.421+0000 7f0592ff5700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-04-30T19:12:20.428+0000 7f0592ff5700 -1 WARNING: the following dangerous and experimental features are enabled: crimson Ignoring malparsed backtrace: next(self.gen) , File "/home/teuthworker/src/github.com_Matan-B_ceph_135726148d97bd4e65d5b006464f156f7132999b/qa/tasks/rados.py", line 286, in task , running.get() Ignoring malparsed backtrace: yield , File "/home/teuthworker/src/git.ceph.com_teuthology_69caab8ec6f44c61c991ae693e8e7b8cd859a896/teuthology/run_tasks.py", line 192, in run_tasks , suppress = manager.__exit__(*exc_info) Ignoring malparsed backtrace: File "src/gevent/greenlet.py", line 906, in gevent._gevent_cgreenlet.Greenlet.run , File "/home/teuthworker/src/github.com_Matan-B_ceph_135726148d97bd4e65d5b006464f156f7132999b/qa/tasks/rados.py", line 274, in thread , run.wait(tests.values()) Ignoring malparsed backtrace: gzip: /var/log/ceph/valgrind/*.gz: No such file or directory , gzip: /var/log/ceph/valgrind/*.gz: No such file or directory , 2023-04-30T20:04:58.384 INFO:tasks.ceph:Archiving crash dumps... Ignoring malparsed backtrace: 2023-04-30T20:05:02.185 DEBUG:teuthology.orchestra.run.smithi032:> test -e /home/ubuntu/cephtest/archive/coredump , 2023-04-30T20:05:02.218 WARNING:teuthology.task.internal:Found coredumps on ubuntu@smithi032.front.sepia.ceph.com, flagging run as failed , 2023-04-30T20:05:02.219 DEBUG:teuthology.orchestra.run.smithi144:> test -e /home/ubuntu/cephtest/archive/coredump Ignoring malparsed backtrace: ceph version 18.0.0-3633-g0ae30b6b (0ae30b6b0a0fdd9fa3bc438c826208438ac27aa2) reef (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7fb1daf3ced4] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x1b0) [0x56211f0270e8] Ignoring malparsed backtrace: 2023-04-30T19:08:26.050 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~33s , 2023-04-30T19:08:27.301 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-04-30T19:08:27.433+0000 7fc7b946d700 -1 WARNING: the following dangerous and experimental features are enabled: crimson Ignoring malparsed backtrace: 2023-04-30T19:09:18.082+0000 7f72e4d0f700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-04-30T19:09:18.289 INFO:teuthology.orchestra.run.smithi032.stdout: , 2023-04-30T19:09:18.289 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":174,"fsid":"4e6dbd7f-aa02-42c5-aed2-840d605d34fe","created":"2023-04-30T19:05:14.879777+0000","modified":"2023-04-30T19:08:47.898007+0000","last_up_change":"2023-04-30T19:08:46.896611+0000","last_in_change":"2023-04-30T19:05:17.167602+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:05:23.030847+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.25,"raw_score_acting":1,"raw_score_stable":1,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:05:25.519290+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2.5,"score_stable":2.5,"optimal_score":0.5,"raw_score_acting":1.25,"raw_score_stable":1.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:05:32.499487+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"157","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":139,"snap_epoch":157,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":2.2599999904632568,"score_stable":2.2599999904632568,"optimal_score":0.5,"raw_score_acting":1.1299999952316284,"raw_score_stable":1.1299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"d6178da2-c707-4fdf-abd5-248d8c097ac6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":173,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6801","nonce":133248}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6800","nonce":133248}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":133248}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6805","nonce":133248}]},"public_addr":"172.21.15.32:6801/133248","cluster_addr":"172.21.15.32:6800/133248","heartbeat_back_addr":"172.21.15.32:6804/133248","heartbeat_front_addr":"172.21.15.32:6805/133248","state":["exists","up"]},{"osd":1,"uuid":"24cda7e5-680d-43f3-9944-7bc94b5a5495","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":173,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":133275}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6803","nonce":133275}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6807","nonce":133275}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":133275}]},"public_addr":"172.21.15.32:6802/133275","cluster_addr":"172.21.15.32:6803/133275","heartbeat_back_addr":"172.21.15.32:6807/133275","heartbeat_front_addr":"172.21.15.32:6806/133275","state":["exists","up"]},{"osd":2,"uuid":"e23153ff-eee1-4ae8-8936-737004527966","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":163,"down_at":173,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":132851}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":132851}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":132851}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":132851}]},"public_addr":"172.21.15.144:6802/132851","cluster_addr":"172.21.15.144:6803/132851","heartbeat_back_addr":"172.21.15.144:6807/132851","heartbeat_front_addr":"172.21.15.144:6806/132851","state":["exists"]},{"osd":3,"uuid":"3c8a1873-5a52-435a-aee4-f7c5611dcf21","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":16,"down_at":163,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":132879}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":132879}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6809","nonce":132879}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6808","nonce":132879}]},"public_addr":"172.21.15.144:6804/132879","cluster_addr":"172.21.15.144:6805/132879","heartbeat_back_addr":"172.21.15.144:6809/132879","heartbeat_front_addr":"172.21.15.144:6808/132879","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"2023-04-30T19:08:46.896611+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2023-04-30T19:08:10.886845+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":1},{"begin":8,"length":3},{"begin":14,"length":1},{"begin":16,"length":2},{"begin":20,"length":1},{"begin":22,"length":4},{"begin":27,"length":1},{"begin":31,"length":2},{"begin":36,"length":3},{"begin":40,"length":1},{"begin":42,"length":2},{"begin":45,"length":1},{"begin":49,"length":1},{"begin":51,"length":2},{"begin":56,"length":1},{"begin":58,"length":2},{"begin":61,"length":1},{"begin":63,"length":1},{"begin":65,"length":5},{"begin":73,"length":1},{"begin":75,"length":4},{"begin":80,"length":2},{"begin":84,"length":1},{"begin":87,"length":1},{"begin":89,"length":2},{"begin":92,"length":2},{"begin":95,"length":3},{"begin":99,"length":3},{"begin":106,"length":1},{"begin":109,"length":2},{"begin":112,"length":1},{"begin":114,"length":1},{"begin":116,"length":2},{"begin":120,"length":1},{"begin":122,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} Ignoring malparsed backtrace: 2023-04-30T19:10:05.106+0000 7fecee6e6700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-04-30T19:10:05.310 INFO:teuthology.orchestra.run.smithi032.stdout: , 2023-04-30T19:10:05.311 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":174,"fsid":"4e6dbd7f-aa02-42c5-aed2-840d605d34fe","created":"2023-04-30T19:05:14.879777+0000","modified":"2023-04-30T19:08:47.898007+0000","last_up_change":"2023-04-30T19:08:46.896611+0000","last_in_change":"2023-04-30T19:05:17.167602+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:05:23.030847+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.25,"raw_score_acting":1,"raw_score_stable":1,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:05:25.519290+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2.5,"score_stable":2.5,"optimal_score":0.5,"raw_score_acting":1.25,"raw_score_stable":1.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:05:32.499487+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"157","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":139,"snap_epoch":157,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":2.2599999904632568,"score_stable":2.2599999904632568,"optimal_score":0.5,"raw_score_acting":1.1299999952316284,"raw_score_stable":1.1299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"d6178da2-c707-4fdf-abd5-248d8c097ac6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":173,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6801","nonce":133248}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6800","nonce":133248}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":133248}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6805","nonce":133248}]},"public_addr":"172.21.15.32:6801/133248","cluster_addr":"172.21.15.32:6800/133248","heartbeat_back_addr":"172.21.15.32:6804/133248","heartbeat_front_addr":"172.21.15.32:6805/133248","state":["exists","up"]},{"osd":1,"uuid":"24cda7e5-680d-43f3-9944-7bc94b5a5495","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":173,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":133275}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6803","nonce":133275}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6807","nonce":133275}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":133275}]},"public_addr":"172.21.15.32:6802/133275","cluster_addr":"172.21.15.32:6803/133275","heartbeat_back_addr":"172.21.15.32:6807/133275","heartbeat_front_addr":"172.21.15.32:6806/133275","state":["exists","up"]},{"osd":2,"uuid":"e23153ff-eee1-4ae8-8936-737004527966","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":163,"down_at":173,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":132851}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":132851}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":132851}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":132851}]},"public_addr":"172.21.15.144:6802/132851","cluster_addr":"172.21.15.144:6803/132851","heartbeat_back_addr":"172.21.15.144:6807/132851","heartbeat_front_addr":"172.21.15.144:6806/132851","state":["exists"]},{"osd":3,"uuid":"3c8a1873-5a52-435a-aee4-f7c5611dcf21","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":16,"down_at":163,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":132879}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":132879}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6809","nonce":132879}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6808","nonce":132879}]},"public_addr":"172.21.15.144:6804/132879","cluster_addr":"172.21.15.144:6805/132879","heartbeat_back_addr":"172.21.15.144:6809/132879","heartbeat_front_addr":"172.21.15.144:6808/132879","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"2023-04-30T19:08:46.896611+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2023-04-30T19:08:10.886845+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":1},{"begin":8,"length":3},{"begin":14,"length":1},{"begin":16,"length":2},{"begin":20,"length":1},{"begin":22,"length":4},{"begin":27,"length":1},{"begin":31,"length":2},{"begin":36,"length":3},{"begin":40,"length":1},{"begin":42,"length":2},{"begin":45,"length":1},{"begin":49,"length":1},{"begin":51,"length":2},{"begin":56,"length":1},{"begin":58,"length":2},{"begin":61,"length":1},{"begin":63,"length":1},{"begin":65,"length":5},{"begin":73,"length":1},{"begin":75,"length":4},{"begin":80,"length":2},{"begin":84,"length":1},{"begin":87,"length":1},{"begin":89,"length":2},{"begin":92,"length":2},{"begin":95,"length":3},{"begin":99,"length":3},{"begin":106,"length":1},{"begin":109,"length":2},{"begin":112,"length":1},{"begin":114,"length":1},{"begin":116,"length":2},{"begin":120,"length":1},{"begin":122,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} Ignoring malparsed backtrace: 2023-04-30T19:10:49.638 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.2 is failed for ~149s , 2023-04-30T19:10:49.638 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~177s , 2023-04-30T19:10:49.911+0000 7fda1187e700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.1 v2:172.21.15.32:6802/133275 for 29sec e174 v174) Ignoring malparsed backtrace: 2023-04-30T19:11:36.582 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-04-30T19:11:36.712+0000 7f03aa294700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-04-30T19:11:36.719+0000 7f03aa294700 -1 WARNING: the following dangerous and experimental features are enabled: crimson Ignoring malparsed backtrace: 2023-04-30T19:12:20.291 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-04-30T19:12:20.421+0000 7f0592ff5700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-04-30T19:12:20.428+0000 7f0592ff5700 -1 WARNING: the following dangerous and experimental features are enabled: crimson Ignoring malparsed backtrace: next(self.gen) , File "/home/teuthworker/src/github.com_Matan-B_ceph_135726148d97bd4e65d5b006464f156f7132999b/qa/tasks/rados.py", line 286, in task , running.get() Ignoring malparsed backtrace: yield , File "/home/teuthworker/src/git.ceph.com_teuthology_69caab8ec6f44c61c991ae693e8e7b8cd859a896/teuthology/run_tasks.py", line 192, in run_tasks , suppress = manager.__exit__(*exc_info) Ignoring malparsed backtrace: File "src/gevent/greenlet.py", line 906, in gevent._gevent_cgreenlet.Greenlet.run , File "/home/teuthworker/src/github.com_Matan-B_ceph_135726148d97bd4e65d5b006464f156f7132999b/qa/tasks/rados.py", line 274, in thread , run.wait(tests.values()) Ignoring malparsed backtrace: gzip: /var/log/ceph/valgrind/*.gz: No such file or directory , gzip: /var/log/ceph/valgrind/*.gz: No such file or directory , 2023-04-30T20:04:58.384 INFO:tasks.ceph:Archiving crash dumps... Ignoring malparsed backtrace: 2023-04-30T20:05:02.185 DEBUG:teuthology.orchestra.run.smithi032:> test -e /home/ubuntu/cephtest/archive/coredump , 2023-04-30T20:05:02.218 WARNING:teuthology.task.internal:Found coredumps on ubuntu@smithi032.front.sepia.ceph.com, flagging run as failed , 2023-04-30T20:05:02.219 DEBUG:teuthology.orchestra.run.smithi144:> test -e /home/ubuntu/cephtest/archive/coredump Ignoring malparsed backtrace: ceph version 18.0.0-3633-g0ae30b6b (0ae30b6b0a0fdd9fa3bc438c826208438ac27aa2) reef (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7fb1daf3ced4] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x1b0) [0x56211f0270e8] Ignoring malparsed backtrace: 2023-04-30T19:08:26.050 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~33s , 2023-04-30T19:08:27.301 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-04-30T19:08:27.433+0000 7fc7b946d700 -1 WARNING: the following dangerous and experimental features are enabled: crimson Ignoring malparsed backtrace: 2023-04-30T19:09:18.082+0000 7f72e4d0f700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-04-30T19:09:18.289 INFO:teuthology.orchestra.run.smithi032.stdout: , 2023-04-30T19:09:18.289 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":174,"fsid":"4e6dbd7f-aa02-42c5-aed2-840d605d34fe","created":"2023-04-30T19:05:14.879777+0000","modified":"2023-04-30T19:08:47.898007+0000","last_up_change":"2023-04-30T19:08:46.896611+0000","last_in_change":"2023-04-30T19:05:17.167602+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:05:23.030847+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.25,"raw_score_acting":1,"raw_score_stable":1,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:05:25.519290+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2.5,"score_stable":2.5,"optimal_score":0.5,"raw_score_acting":1.25,"raw_score_stable":1.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:05:32.499487+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"157","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":139,"snap_epoch":157,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":2.2599999904632568,"score_stable":2.2599999904632568,"optimal_score":0.5,"raw_score_acting":1.1299999952316284,"raw_score_stable":1.1299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"d6178da2-c707-4fdf-abd5-248d8c097ac6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":173,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6801","nonce":133248}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6800","nonce":133248}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":133248}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6805","nonce":133248}]},"public_addr":"172.21.15.32:6801/133248","cluster_addr":"172.21.15.32:6800/133248","heartbeat_back_addr":"172.21.15.32:6804/133248","heartbeat_front_addr":"172.21.15.32:6805/133248","state":["exists","up"]},{"osd":1,"uuid":"24cda7e5-680d-43f3-9944-7bc94b5a5495","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":173,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":133275}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6803","nonce":133275}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6807","nonce":133275}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":133275}]},"public_addr":"172.21.15.32:6802/133275","cluster_addr":"172.21.15.32:6803/133275","heartbeat_back_addr":"172.21.15.32:6807/133275","heartbeat_front_addr":"172.21.15.32:6806/133275","state":["exists","up"]},{"osd":2,"uuid":"e23153ff-eee1-4ae8-8936-737004527966","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":163,"down_at":173,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":132851}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":132851}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":132851}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":132851}]},"public_addr":"172.21.15.144:6802/132851","cluster_addr":"172.21.15.144:6803/132851","heartbeat_back_addr":"172.21.15.144:6807/132851","heartbeat_front_addr":"172.21.15.144:6806/132851","state":["exists"]},{"osd":3,"uuid":"3c8a1873-5a52-435a-aee4-f7c5611dcf21","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":16,"down_at":163,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":132879}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":132879}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6809","nonce":132879}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6808","nonce":132879}]},"public_addr":"172.21.15.144:6804/132879","cluster_addr":"172.21.15.144:6805/132879","heartbeat_back_addr":"172.21.15.144:6809/132879","heartbeat_front_addr":"172.21.15.144:6808/132879","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"2023-04-30T19:08:46.896611+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2023-04-30T19:08:10.886845+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":1},{"begin":8,"length":3},{"begin":14,"length":1},{"begin":16,"length":2},{"begin":20,"length":1},{"begin":22,"length":4},{"begin":27,"length":1},{"begin":31,"length":2},{"begin":36,"length":3},{"begin":40,"length":1},{"begin":42,"length":2},{"begin":45,"length":1},{"begin":49,"length":1},{"begin":51,"length":2},{"begin":56,"length":1},{"begin":58,"length":2},{"begin":61,"length":1},{"begin":63,"length":1},{"begin":65,"length":5},{"begin":73,"length":1},{"begin":75,"length":4},{"begin":80,"length":2},{"begin":84,"length":1},{"begin":87,"length":1},{"begin":89,"length":2},{"begin":92,"length":2},{"begin":95,"length":3},{"begin":99,"length":3},{"begin":106,"length":1},{"begin":109,"length":2},{"begin":112,"length":1},{"begin":114,"length":1},{"begin":116,"length":2},{"begin":120,"length":1},{"begin":122,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} Ignoring malparsed backtrace: 2023-04-30T19:10:05.106+0000 7fecee6e6700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-04-30T19:10:05.310 INFO:teuthology.orchestra.run.smithi032.stdout: , 2023-04-30T19:10:05.311 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":174,"fsid":"4e6dbd7f-aa02-42c5-aed2-840d605d34fe","created":"2023-04-30T19:05:14.879777+0000","modified":"2023-04-30T19:08:47.898007+0000","last_up_change":"2023-04-30T19:08:46.896611+0000","last_in_change":"2023-04-30T19:05:17.167602+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:05:23.030847+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.25,"raw_score_acting":1,"raw_score_stable":1,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:05:25.519290+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2.5,"score_stable":2.5,"optimal_score":0.5,"raw_score_acting":1.25,"raw_score_stable":1.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:05:32.499487+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"157","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":139,"snap_epoch":157,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":2.2599999904632568,"score_stable":2.2599999904632568,"optimal_score":0.5,"raw_score_acting":1.1299999952316284,"raw_score_stable":1.1299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"d6178da2-c707-4fdf-abd5-248d8c097ac6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":173,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6801","nonce":133248}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6800","nonce":133248}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":133248}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6805","nonce":133248}]},"public_addr":"172.21.15.32:6801/133248","cluster_addr":"172.21.15.32:6800/133248","heartbeat_back_addr":"172.21.15.32:6804/133248","heartbeat_front_addr":"172.21.15.32:6805/133248","state":["exists","up"]},{"osd":1,"uuid":"24cda7e5-680d-43f3-9944-7bc94b5a5495","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":173,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":133275}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6803","nonce":133275}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6807","nonce":133275}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":133275}]},"public_addr":"172.21.15.32:6802/133275","cluster_addr":"172.21.15.32:6803/133275","heartbeat_back_addr":"172.21.15.32:6807/133275","heartbeat_front_addr":"172.21.15.32:6806/133275","state":["exists","up"]},{"osd":2,"uuid":"e23153ff-eee1-4ae8-8936-737004527966","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":163,"down_at":173,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":132851}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":132851}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":132851}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":132851}]},"public_addr":"172.21.15.144:6802/132851","cluster_addr":"172.21.15.144:6803/132851","heartbeat_back_addr":"172.21.15.144:6807/132851","heartbeat_front_addr":"172.21.15.144:6806/132851","state":["exists"]},{"osd":3,"uuid":"3c8a1873-5a52-435a-aee4-f7c5611dcf21","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":16,"down_at":163,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":132879}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":132879}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6809","nonce":132879}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6808","nonce":132879}]},"public_addr":"172.21.15.144:6804/132879","cluster_addr":"172.21.15.144:6805/132879","heartbeat_back_addr":"172.21.15.144:6809/132879","heartbeat_front_addr":"172.21.15.144:6808/132879","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"2023-04-30T19:08:46.896611+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2023-04-30T19:08:10.886845+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":1},{"begin":8,"length":3},{"begin":14,"length":1},{"begin":16,"length":2},{"begin":20,"length":1},{"begin":22,"length":4},{"begin":27,"length":1},{"begin":31,"length":2},{"begin":36,"length":3},{"begin":40,"length":1},{"begin":42,"length":2},{"begin":45,"length":1},{"begin":49,"length":1},{"begin":51,"length":2},{"begin":56,"length":1},{"begin":58,"length":2},{"begin":61,"length":1},{"begin":63,"length":1},{"begin":65,"length":5},{"begin":73,"length":1},{"begin":75,"length":4},{"begin":80,"length":2},{"begin":84,"length":1},{"begin":87,"length":1},{"begin":89,"length":2},{"begin":92,"length":2},{"begin":95,"length":3},{"begin":99,"length":3},{"begin":106,"length":1},{"begin":109,"length":2},{"begin":112,"length":1},{"begin":114,"length":1},{"begin":116,"length":2},{"begin":120,"length":1},{"begin":122,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} Ignoring malparsed backtrace: 2023-04-30T19:10:49.638 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.2 is failed for ~149s , 2023-04-30T19:10:49.638 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~177s , 2023-04-30T19:10:49.911+0000 7fda1187e700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.1 v2:172.21.15.32:6802/133275 for 29sec e174 v174) Ignoring malparsed backtrace: 2023-04-30T19:11:36.582 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-04-30T19:11:36.712+0000 7f03aa294700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-04-30T19:11:36.719+0000 7f03aa294700 -1 WARNING: the following dangerous and experimental features are enabled: crimson Ignoring malparsed backtrace: 2023-04-30T19:12:20.291 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-04-30T19:12:20.421+0000 7f0592ff5700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-04-30T19:12:20.428+0000 7f0592ff5700 -1 WARNING: the following dangerous and experimental features are enabled: crimson Ignoring malparsed backtrace: next(self.gen) , File "/home/teuthworker/src/github.com_Matan-B_ceph_135726148d97bd4e65d5b006464f156f7132999b/qa/tasks/rados.py", line 286, in task , running.get() Ignoring malparsed backtrace: yield , File "/home/teuthworker/src/git.ceph.com_teuthology_69caab8ec6f44c61c991ae693e8e7b8cd859a896/teuthology/run_tasks.py", line 192, in run_tasks , suppress = manager.__exit__(*exc_info) Ignoring malparsed backtrace: File "src/gevent/greenlet.py", line 906, in gevent._gevent_cgreenlet.Greenlet.run , File "/home/teuthworker/src/github.com_Matan-B_ceph_135726148d97bd4e65d5b006464f156f7132999b/qa/tasks/rados.py", line 274, in thread , run.wait(tests.values()) Ignoring malparsed backtrace: gzip: /var/log/ceph/valgrind/*.gz: No such file or directory , gzip: /var/log/ceph/valgrind/*.gz: No such file or directory , 2023-04-30T20:04:58.384 INFO:tasks.ceph:Archiving crash dumps... Ignoring malparsed backtrace: 2023-04-30T20:05:02.185 DEBUG:teuthology.orchestra.run.smithi032:> test -e /home/ubuntu/cephtest/archive/coredump , 2023-04-30T20:05:02.218 WARNING:teuthology.task.internal:Found coredumps on ubuntu@smithi032.front.sepia.ceph.com, flagging run as failed , 2023-04-30T20:05:02.219 DEBUG:teuthology.orchestra.run.smithi144:> test -e /home/ubuntu/cephtest/archive/coredump Ignoring malparsed backtrace: ceph version 18.0.0-3633-g0ae30b6b (0ae30b6b0a0fdd9fa3bc438c826208438ac27aa2) reef (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7fb1daf3ced4] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x1b0) [0x56211f0270e8] Ignoring malparsed backtrace: 2023-04-30T19:08:26.050 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~33s , 2023-04-30T19:08:27.301 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-04-30T19:08:27.433+0000 7fc7b946d700 -1 WARNING: the following dangerous and experimental features are enabled: crimson Ignoring malparsed backtrace: 2023-04-30T19:09:18.082+0000 7f72e4d0f700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-04-30T19:09:18.289 INFO:teuthology.orchestra.run.smithi032.stdout: , 2023-04-30T19:09:18.289 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":174,"fsid":"4e6dbd7f-aa02-42c5-aed2-840d605d34fe","created":"2023-04-30T19:05:14.879777+0000","modified":"2023-04-30T19:08:47.898007+0000","last_up_change":"2023-04-30T19:08:46.896611+0000","last_in_change":"2023-04-30T19:05:17.167602+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:05:23.030847+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.25,"raw_score_acting":1,"raw_score_stable":1,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:05:25.519290+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2.5,"score_stable":2.5,"optimal_score":0.5,"raw_score_acting":1.25,"raw_score_stable":1.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:05:32.499487+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"157","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":139,"snap_epoch":157,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":2.2599999904632568,"score_stable":2.2599999904632568,"optimal_score":0.5,"raw_score_acting":1.1299999952316284,"raw_score_stable":1.1299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"d6178da2-c707-4fdf-abd5-248d8c097ac6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":173,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6801","nonce":133248}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6800","nonce":133248}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":133248}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6805","nonce":133248}]},"public_addr":"172.21.15.32:6801/133248","cluster_addr":"172.21.15.32:6800/133248","heartbeat_back_addr":"172.21.15.32:6804/133248","heartbeat_front_addr":"172.21.15.32:6805/133248","state":["exists","up"]},{"osd":1,"uuid":"24cda7e5-680d-43f3-9944-7bc94b5a5495","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":173,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":133275}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6803","nonce":133275}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6807","nonce":133275}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":133275}]},"public_addr":"172.21.15.32:6802/133275","cluster_addr":"172.21.15.32:6803/133275","heartbeat_back_addr":"172.21.15.32:6807/133275","heartbeat_front_addr":"172.21.15.32:6806/133275","state":["exists","up"]},{"osd":2,"uuid":"e23153ff-eee1-4ae8-8936-737004527966","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":163,"down_at":173,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":132851}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":132851}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":132851}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":132851}]},"public_addr":"172.21.15.144:6802/132851","cluster_addr":"172.21.15.144:6803/132851","heartbeat_back_addr":"172.21.15.144:6807/132851","heartbeat_front_addr":"172.21.15.144:6806/132851","state":["exists"]},{"osd":3,"uuid":"3c8a1873-5a52-435a-aee4-f7c5611dcf21","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":16,"down_at":163,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":132879}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":132879}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6809","nonce":132879}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6808","nonce":132879}]},"public_addr":"172.21.15.144:6804/132879","cluster_addr":"172.21.15.144:6805/132879","heartbeat_back_addr":"172.21.15.144:6809/132879","heartbeat_front_addr":"172.21.15.144:6808/132879","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"2023-04-30T19:08:46.896611+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2023-04-30T19:08:10.886845+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":1},{"begin":8,"length":3},{"begin":14,"length":1},{"begin":16,"length":2},{"begin":20,"length":1},{"begin":22,"length":4},{"begin":27,"length":1},{"begin":31,"length":2},{"begin":36,"length":3},{"begin":40,"length":1},{"begin":42,"length":2},{"begin":45,"length":1},{"begin":49,"length":1},{"begin":51,"length":2},{"begin":56,"length":1},{"begin":58,"length":2},{"begin":61,"length":1},{"begin":63,"length":1},{"begin":65,"length":5},{"begin":73,"length":1},{"begin":75,"length":4},{"begin":80,"length":2},{"begin":84,"length":1},{"begin":87,"length":1},{"begin":89,"length":2},{"begin":92,"length":2},{"begin":95,"length":3},{"begin":99,"length":3},{"begin":106,"length":1},{"begin":109,"length":2},{"begin":112,"length":1},{"begin":114,"length":1},{"begin":116,"length":2},{"begin":120,"length":1},{"begin":122,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} Ignoring malparsed backtrace: 2023-04-30T19:10:05.106+0000 7fecee6e6700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-04-30T19:10:05.310 INFO:teuthology.orchestra.run.smithi032.stdout: , 2023-04-30T19:10:05.311 INFO:teuthology.orchestra.run.smithi032.stdout:{"epoch":174,"fsid":"4e6dbd7f-aa02-42c5-aed2-840d605d34fe","created":"2023-04-30T19:05:14.879777+0000","modified":"2023-04-30T19:08:47.898007+0000","last_up_change":"2023-04-30T19:08:46.896611+0000","last_in_change":"2023-04-30T19:05:17.167602+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-04-30T19:05:23.030847+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.25,"raw_score_acting":1,"raw_score_stable":1,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-04-30T19:05:25.519290+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2.5,"score_stable":2.5,"optimal_score":0.5,"raw_score_acting":1.25,"raw_score_stable":1.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-04-30T19:05:32.499487+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"157","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":139,"snap_epoch":157,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":2.2599999904632568,"score_stable":2.2599999904632568,"optimal_score":0.5,"raw_score_acting":1.1299999952316284,"raw_score_stable":1.1299999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"d6178da2-c707-4fdf-abd5-248d8c097ac6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":173,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6801","nonce":133248}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6800","nonce":133248}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6804","nonce":133248}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6805","nonce":133248}]},"public_addr":"172.21.15.32:6801/133248","cluster_addr":"172.21.15.32:6800/133248","heartbeat_back_addr":"172.21.15.32:6804/133248","heartbeat_front_addr":"172.21.15.32:6805/133248","state":["exists","up"]},{"osd":1,"uuid":"24cda7e5-680d-43f3-9944-7bc94b5a5495","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":173,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6802","nonce":133275}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6803","nonce":133275}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6807","nonce":133275}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.32:6806","nonce":133275}]},"public_addr":"172.21.15.32:6802/133275","cluster_addr":"172.21.15.32:6803/133275","heartbeat_back_addr":"172.21.15.32:6807/133275","heartbeat_front_addr":"172.21.15.32:6806/133275","state":["exists","up"]},{"osd":2,"uuid":"e23153ff-eee1-4ae8-8936-737004527966","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":163,"down_at":173,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":132851}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":132851}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":132851}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":132851}]},"public_addr":"172.21.15.144:6802/132851","cluster_addr":"172.21.15.144:6803/132851","heartbeat_back_addr":"172.21.15.144:6807/132851","heartbeat_front_addr":"172.21.15.144:6806/132851","state":["exists"]},{"osd":3,"uuid":"3c8a1873-5a52-435a-aee4-f7c5611dcf21","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":16,"down_at":163,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":132879}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":132879}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6809","nonce":132879}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6808","nonce":132879}]},"public_addr":"172.21.15.144:6804/132879","cluster_addr":"172.21.15.144:6805/132879","heartbeat_back_addr":"172.21.15.144:6809/132879","heartbeat_front_addr":"172.21.15.144:6808/132879","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"2023-04-30T19:08:46.896611+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2023-04-30T19:08:10.886845+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":1},{"begin":8,"length":3},{"begin":14,"length":1},{"begin":16,"length":2},{"begin":20,"length":1},{"begin":22,"length":4},{"begin":27,"length":1},{"begin":31,"length":2},{"begin":36,"length":3},{"begin":40,"length":1},{"begin":42,"length":2},{"begin":45,"length":1},{"begin":49,"length":1},{"begin":51,"length":2},{"begin":56,"length":1},{"begin":58,"length":2},{"begin":61,"length":1},{"begin":63,"length":1},{"begin":65,"length":5},{"begin":73,"length":1},{"begin":75,"length":4},{"begin":80,"length":2},{"begin":84,"length":1},{"begin":87,"length":1},{"begin":89,"length":2},{"begin":92,"length":2},{"begin":95,"length":3},{"begin":99,"length":3},{"begin":106,"length":1},{"begin":109,"length":2},{"begin":112,"length":1},{"begin":114,"length":1},{"begin":116,"length":2},{"begin":120,"length":1},{"begin":122,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} Ignoring malparsed backtrace: 2023-04-30T19:10:49.638 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.2 is failed for ~149s , 2023-04-30T19:10:49.638 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.3 is failed for ~177s , 2023-04-30T19:10:49.911+0000 7fda1187e700 -1 mon.a@0(leader) e1 get_health_metrics reporting 1 slow ops, oldest is osd_failure(failed timeout osd.1 v2:172.21.15.32:6802/133275 for 29sec e174 v174) Ignoring malparsed backtrace: 2023-04-30T19:11:36.582 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-04-30T19:11:36.712+0000 7f03aa294700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-04-30T19:11:36.719+0000 7f03aa294700 -1 WARNING: the following dangerous and experimental features are enabled: crimson Ignoring malparsed backtrace: 2023-04-30T19:12:20.291 DEBUG:teuthology.orchestra.run.smithi032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-04-30T19:12:20.421+0000 7f0592ff5700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-04-30T19:12:20.428+0000 7f0592ff5700 -1 WARNING: the following dangerous and experimental features are enabled: crimson Ignoring malparsed backtrace: next(self.gen) , File "/home/teuthworker/src/github.com_Matan-B_ceph_135726148d97bd4e65d5b006464f156f7132999b/qa/tasks/rados.py", line 286, in task , running.get() Ignoring malparsed backtrace: yield , File "/home/teuthworker/src/git.ceph.com_teuthology_69caab8ec6f44c61c991ae693e8e7b8cd859a896/teuthology/run_tasks.py", line 192, in run_tasks , suppress = manager.__exit__(*exc_info) Ignoring malparsed backtrace: File "src/gevent/greenlet.py", line 906, in gevent._gevent_cgreenlet.Greenlet.run , File "/home/teuthworker/src/github.com_Matan-B_ceph_135726148d97bd4e65d5b006464f156f7132999b/qa/tasks/rados.py", line 274, in thread , run.wait(tests.values()) Ignoring malparsed backtrace: gzip: /var/log/ceph/valgrind/*.gz: No such file or directory , gzip: /var/log/ceph/valgrind/*.gz: No such file or directory , 2023-04-30T20:04:58.384 INFO:tasks.ceph:Archiving crash dumps... Ignoring malparsed backtrace: 2023-04-30T20:05:02.185 DEBUG:teuthology.orchestra.run.smithi032:> test -e /home/ubuntu/cephtest/archive/coredump , 2023-04-30T20:05:02.218 WARNING:teuthology.task.internal:Found coredumps on ubuntu@smithi032.front.sepia.ceph.com, flagging run as failed , 2023-04-30T20:05:02.219 DEBUG:teuthology.orchestra.run.smithi144:> test -e /home/ubuntu/cephtest/archive/coredump Ignoring malparsed backtrace: ceph version 18.0.0-3633-g0ae30b6b (0ae30b6b0a0fdd9fa3bc438c826208438ac27aa2) reef (dev) , 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x120) [0x7f9b219d07ff] , 2: (ceph::register_assert_context(ceph::common::CephContext*)+0) [0x7f9b219d0a2a] Ignoring malparsed backtrace: instructed osd(s) 0 to deep-scrub , 2023-04-30T19:20:36.407 INFO:tasks.ceph:Scrubbing osd.1 , 2023-04-30T19:20:36.408 DEBUG:teuthology.orchestra.run.smithi165:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.1 config set osd_debug_deep_scrub_sleep 0 Ignoring malparsed backtrace: 2023-04-30T19:20:37.927 INFO:tasks.ceph:pgid 6.12 last_scrub_stamp 2023-04-30T18:55:02.438577+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=2, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.927 INFO:tasks.ceph:pgid 5.11 last_scrub_stamp 2023-04-30T18:54:56.415471+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=54, tm_sec=56, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.928 INFO:tasks.ceph:pgid 4.10 last_scrub_stamp 2023-04-30T18:54:50.398271+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=54, tm_sec=50, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:20:37.957 INFO:tasks.ceph:pgid 5.14 last_scrub_stamp 2023-04-30T18:54:56.415471+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=54, tm_sec=56, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.957 INFO:tasks.ceph:pgid 4.15 last_scrub_stamp 2023-04-30T18:54:50.398271+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=54, tm_sec=50, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.957 INFO:tasks.ceph:pgid 8.19 last_scrub_stamp 2023-04-30T18:55:31.335752+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=31, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:20:37.986 INFO:tasks.ceph:pgid 8.6 last_scrub_stamp 2023-04-30T18:55:31.335752+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=31, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.986 INFO:tasks.ceph:pgid 9.7 last_scrub_stamp 2023-04-30T18:55:55.391997+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=55, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.986 INFO:tasks.ceph:pgid 10.4 last_scrub_stamp 2023-04-30T18:56:43.320182+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=56, tm_sec=43, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:20:38.015 INFO:tasks.ceph:pgid 9.2 last_scrub_stamp 2023-04-30T18:55:55.391997+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=55, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.015 INFO:tasks.ceph:pgid 10.1 last_scrub_stamp 2023-04-30T18:56:43.320182+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=56, tm_sec=43, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.015 INFO:tasks.ceph:pgid 11.0 last_scrub_stamp 2023-04-30T18:57:00.371165+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=57, tm_sec=0, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:20:38.044 INFO:tasks.ceph:pgid 7.5 last_scrub_stamp 2023-04-30T18:55:17.299077+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=17, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.044 INFO:tasks.ceph:pgid 6.4 last_scrub_stamp 2023-04-30T18:55:02.438577+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=2, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.044 INFO:tasks.ceph:pgid 5.7 last_scrub_stamp 2023-04-30T18:54:56.415471+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=54, tm_sec=56, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:20:38.073 INFO:tasks.ceph:pgid 15.10 last_scrub_stamp 2023-04-30T19:07:46.490515+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=7, tm_sec=46, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.073 INFO:tasks.ceph:pgid 14.11 last_scrub_stamp 2023-04-30T19:05:30.431322+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=5, tm_sec=30, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.073 INFO:tasks.ceph:pgid 13.12 last_scrub_stamp 2023-04-30T18:57:33.358860+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=57, tm_sec=33, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:21:17.308 INFO:tasks.ceph.mgr.x:Stopped , 2023-04-30T19:21:17.308 INFO:teuthology.misc:Shutting down mon daemons... , 2023-04-30T19:21:17.309 DEBUG:tasks.ceph.mon.a:waiting for process to exit Ignoring malparsed backtrace: next(self.gen) , File "/home/teuthworker/src/git.ceph.com_teuthology_69caab8ec6f44c61c991ae693e8e7b8cd859a896/teuthology/task/internal/__init__.py", line 420, in archive , fetch_binaries_for_coredumps(path, rem) Ignoring malparsed backtrace: ceph version 18.0.0-3633-g0ae30b6b (0ae30b6b0a0fdd9fa3bc438c826208438ac27aa2) reef (dev) , 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x120) [0x7f9b219d07ff] , 2: (ceph::register_assert_context(ceph::common::CephContext*)+0) [0x7f9b219d0a2a] Ignoring malparsed backtrace: instructed osd(s) 0 to deep-scrub , 2023-04-30T19:20:36.407 INFO:tasks.ceph:Scrubbing osd.1 , 2023-04-30T19:20:36.408 DEBUG:teuthology.orchestra.run.smithi165:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.1 config set osd_debug_deep_scrub_sleep 0 Ignoring malparsed backtrace: 2023-04-30T19:20:37.927 INFO:tasks.ceph:pgid 6.12 last_scrub_stamp 2023-04-30T18:55:02.438577+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=2, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.927 INFO:tasks.ceph:pgid 5.11 last_scrub_stamp 2023-04-30T18:54:56.415471+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=54, tm_sec=56, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.928 INFO:tasks.ceph:pgid 4.10 last_scrub_stamp 2023-04-30T18:54:50.398271+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=54, tm_sec=50, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:20:37.957 INFO:tasks.ceph:pgid 5.14 last_scrub_stamp 2023-04-30T18:54:56.415471+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=54, tm_sec=56, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.957 INFO:tasks.ceph:pgid 4.15 last_scrub_stamp 2023-04-30T18:54:50.398271+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=54, tm_sec=50, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.957 INFO:tasks.ceph:pgid 8.19 last_scrub_stamp 2023-04-30T18:55:31.335752+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=31, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:20:37.986 INFO:tasks.ceph:pgid 8.6 last_scrub_stamp 2023-04-30T18:55:31.335752+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=31, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.986 INFO:tasks.ceph:pgid 9.7 last_scrub_stamp 2023-04-30T18:55:55.391997+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=55, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.986 INFO:tasks.ceph:pgid 10.4 last_scrub_stamp 2023-04-30T18:56:43.320182+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=56, tm_sec=43, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:20:38.015 INFO:tasks.ceph:pgid 9.2 last_scrub_stamp 2023-04-30T18:55:55.391997+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=55, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.015 INFO:tasks.ceph:pgid 10.1 last_scrub_stamp 2023-04-30T18:56:43.320182+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=56, tm_sec=43, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.015 INFO:tasks.ceph:pgid 11.0 last_scrub_stamp 2023-04-30T18:57:00.371165+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=57, tm_sec=0, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:20:38.044 INFO:tasks.ceph:pgid 7.5 last_scrub_stamp 2023-04-30T18:55:17.299077+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=17, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.044 INFO:tasks.ceph:pgid 6.4 last_scrub_stamp 2023-04-30T18:55:02.438577+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=2, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.044 INFO:tasks.ceph:pgid 5.7 last_scrub_stamp 2023-04-30T18:54:56.415471+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=54, tm_sec=56, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:20:38.073 INFO:tasks.ceph:pgid 15.10 last_scrub_stamp 2023-04-30T19:07:46.490515+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=7, tm_sec=46, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.073 INFO:tasks.ceph:pgid 14.11 last_scrub_stamp 2023-04-30T19:05:30.431322+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=5, tm_sec=30, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.073 INFO:tasks.ceph:pgid 13.12 last_scrub_stamp 2023-04-30T18:57:33.358860+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=57, tm_sec=33, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:21:17.308 INFO:tasks.ceph.mgr.x:Stopped , 2023-04-30T19:21:17.308 INFO:teuthology.misc:Shutting down mon daemons... , 2023-04-30T19:21:17.309 DEBUG:tasks.ceph.mon.a:waiting for process to exit Ignoring malparsed backtrace: next(self.gen) , File "/home/teuthworker/src/git.ceph.com_teuthology_69caab8ec6f44c61c991ae693e8e7b8cd859a896/teuthology/task/internal/__init__.py", line 420, in archive , fetch_binaries_for_coredumps(path, rem) Ignoring malparsed backtrace: ceph version 18.0.0-3633-g0ae30b6b (0ae30b6b0a0fdd9fa3bc438c826208438ac27aa2) reef (dev) , 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x120) [0x7f9b219d07ff] , 2: (ceph::register_assert_context(ceph::common::CephContext*)+0) [0x7f9b219d0a2a] Ignoring malparsed backtrace: instructed osd(s) 0 to deep-scrub , 2023-04-30T19:20:36.407 INFO:tasks.ceph:Scrubbing osd.1 , 2023-04-30T19:20:36.408 DEBUG:teuthology.orchestra.run.smithi165:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.1 config set osd_debug_deep_scrub_sleep 0 Ignoring malparsed backtrace: 2023-04-30T19:20:37.927 INFO:tasks.ceph:pgid 6.12 last_scrub_stamp 2023-04-30T18:55:02.438577+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=2, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.927 INFO:tasks.ceph:pgid 5.11 last_scrub_stamp 2023-04-30T18:54:56.415471+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=54, tm_sec=56, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.928 INFO:tasks.ceph:pgid 4.10 last_scrub_stamp 2023-04-30T18:54:50.398271+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=54, tm_sec=50, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:20:37.957 INFO:tasks.ceph:pgid 5.14 last_scrub_stamp 2023-04-30T18:54:56.415471+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=54, tm_sec=56, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.957 INFO:tasks.ceph:pgid 4.15 last_scrub_stamp 2023-04-30T18:54:50.398271+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=54, tm_sec=50, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.957 INFO:tasks.ceph:pgid 8.19 last_scrub_stamp 2023-04-30T18:55:31.335752+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=31, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:20:37.986 INFO:tasks.ceph:pgid 8.6 last_scrub_stamp 2023-04-30T18:55:31.335752+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=31, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.986 INFO:tasks.ceph:pgid 9.7 last_scrub_stamp 2023-04-30T18:55:55.391997+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=55, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.986 INFO:tasks.ceph:pgid 10.4 last_scrub_stamp 2023-04-30T18:56:43.320182+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=56, tm_sec=43, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:20:38.015 INFO:tasks.ceph:pgid 9.2 last_scrub_stamp 2023-04-30T18:55:55.391997+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=55, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.015 INFO:tasks.ceph:pgid 10.1 last_scrub_stamp 2023-04-30T18:56:43.320182+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=56, tm_sec=43, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.015 INFO:tasks.ceph:pgid 11.0 last_scrub_stamp 2023-04-30T18:57:00.371165+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=57, tm_sec=0, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:20:38.044 INFO:tasks.ceph:pgid 7.5 last_scrub_stamp 2023-04-30T18:55:17.299077+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=17, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.044 INFO:tasks.ceph:pgid 6.4 last_scrub_stamp 2023-04-30T18:55:02.438577+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=2, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.044 INFO:tasks.ceph:pgid 5.7 last_scrub_stamp 2023-04-30T18:54:56.415471+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=54, tm_sec=56, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:20:38.073 INFO:tasks.ceph:pgid 15.10 last_scrub_stamp 2023-04-30T19:07:46.490515+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=7, tm_sec=46, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.073 INFO:tasks.ceph:pgid 14.11 last_scrub_stamp 2023-04-30T19:05:30.431322+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=5, tm_sec=30, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.073 INFO:tasks.ceph:pgid 13.12 last_scrub_stamp 2023-04-30T18:57:33.358860+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=57, tm_sec=33, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:21:17.308 INFO:tasks.ceph.mgr.x:Stopped , 2023-04-30T19:21:17.308 INFO:teuthology.misc:Shutting down mon daemons... , 2023-04-30T19:21:17.309 DEBUG:tasks.ceph.mon.a:waiting for process to exit Ignoring malparsed backtrace: next(self.gen) , File "/home/teuthworker/src/git.ceph.com_teuthology_69caab8ec6f44c61c991ae693e8e7b8cd859a896/teuthology/task/internal/__init__.py", line 420, in archive , fetch_binaries_for_coredumps(path, rem) Ignoring malparsed backtrace: ceph version 18.0.0-3633-g0ae30b6b (0ae30b6b0a0fdd9fa3bc438c826208438ac27aa2) reef (dev) , 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x120) [0x7f9b219d07ff] , 2: (ceph::register_assert_context(ceph::common::CephContext*)+0) [0x7f9b219d0a2a] Ignoring malparsed backtrace: instructed osd(s) 0 to deep-scrub , 2023-04-30T19:20:36.407 INFO:tasks.ceph:Scrubbing osd.1 , 2023-04-30T19:20:36.408 DEBUG:teuthology.orchestra.run.smithi165:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.1 config set osd_debug_deep_scrub_sleep 0 Ignoring malparsed backtrace: 2023-04-30T19:20:37.927 INFO:tasks.ceph:pgid 6.12 last_scrub_stamp 2023-04-30T18:55:02.438577+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=2, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.927 INFO:tasks.ceph:pgid 5.11 last_scrub_stamp 2023-04-30T18:54:56.415471+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=54, tm_sec=56, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.928 INFO:tasks.ceph:pgid 4.10 last_scrub_stamp 2023-04-30T18:54:50.398271+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=54, tm_sec=50, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:20:37.957 INFO:tasks.ceph:pgid 5.14 last_scrub_stamp 2023-04-30T18:54:56.415471+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=54, tm_sec=56, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.957 INFO:tasks.ceph:pgid 4.15 last_scrub_stamp 2023-04-30T18:54:50.398271+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=54, tm_sec=50, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.957 INFO:tasks.ceph:pgid 8.19 last_scrub_stamp 2023-04-30T18:55:31.335752+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=31, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:20:37.986 INFO:tasks.ceph:pgid 8.6 last_scrub_stamp 2023-04-30T18:55:31.335752+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=31, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.986 INFO:tasks.ceph:pgid 9.7 last_scrub_stamp 2023-04-30T18:55:55.391997+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=55, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.986 INFO:tasks.ceph:pgid 10.4 last_scrub_stamp 2023-04-30T18:56:43.320182+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=56, tm_sec=43, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:20:38.015 INFO:tasks.ceph:pgid 9.2 last_scrub_stamp 2023-04-30T18:55:55.391997+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=55, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.015 INFO:tasks.ceph:pgid 10.1 last_scrub_stamp 2023-04-30T18:56:43.320182+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=56, tm_sec=43, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.015 INFO:tasks.ceph:pgid 11.0 last_scrub_stamp 2023-04-30T18:57:00.371165+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=57, tm_sec=0, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:20:38.044 INFO:tasks.ceph:pgid 7.5 last_scrub_stamp 2023-04-30T18:55:17.299077+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=17, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.044 INFO:tasks.ceph:pgid 6.4 last_scrub_stamp 2023-04-30T18:55:02.438577+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=2, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.044 INFO:tasks.ceph:pgid 5.7 last_scrub_stamp 2023-04-30T18:54:56.415471+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=54, tm_sec=56, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:20:38.073 INFO:tasks.ceph:pgid 15.10 last_scrub_stamp 2023-04-30T19:07:46.490515+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=7, tm_sec=46, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.073 INFO:tasks.ceph:pgid 14.11 last_scrub_stamp 2023-04-30T19:05:30.431322+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=5, tm_sec=30, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.073 INFO:tasks.ceph:pgid 13.12 last_scrub_stamp 2023-04-30T18:57:33.358860+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=57, tm_sec=33, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:21:17.308 INFO:tasks.ceph.mgr.x:Stopped , 2023-04-30T19:21:17.308 INFO:teuthology.misc:Shutting down mon daemons... , 2023-04-30T19:21:17.309 DEBUG:tasks.ceph.mon.a:waiting for process to exit Ignoring malparsed backtrace: next(self.gen) , File "/home/teuthworker/src/git.ceph.com_teuthology_69caab8ec6f44c61c991ae693e8e7b8cd859a896/teuthology/task/internal/__init__.py", line 420, in archive , fetch_binaries_for_coredumps(path, rem) Ignoring malparsed backtrace: ceph version 18.0.0-3633-g0ae30b6b (0ae30b6b0a0fdd9fa3bc438c826208438ac27aa2) reef (dev) , 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x120) [0x7f9b219d07ff] , 2: (ceph::register_assert_context(ceph::common::CephContext*)+0) [0x7f9b219d0a2a] Ignoring malparsed backtrace: instructed osd(s) 0 to deep-scrub , 2023-04-30T19:20:36.407 INFO:tasks.ceph:Scrubbing osd.1 , 2023-04-30T19:20:36.408 DEBUG:teuthology.orchestra.run.smithi165:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.1 config set osd_debug_deep_scrub_sleep 0 Ignoring malparsed backtrace: 2023-04-30T19:20:37.927 INFO:tasks.ceph:pgid 6.12 last_scrub_stamp 2023-04-30T18:55:02.438577+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=2, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.927 INFO:tasks.ceph:pgid 5.11 last_scrub_stamp 2023-04-30T18:54:56.415471+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=54, tm_sec=56, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.928 INFO:tasks.ceph:pgid 4.10 last_scrub_stamp 2023-04-30T18:54:50.398271+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=54, tm_sec=50, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:20:37.957 INFO:tasks.ceph:pgid 5.14 last_scrub_stamp 2023-04-30T18:54:56.415471+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=54, tm_sec=56, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.957 INFO:tasks.ceph:pgid 4.15 last_scrub_stamp 2023-04-30T18:54:50.398271+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=54, tm_sec=50, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.957 INFO:tasks.ceph:pgid 8.19 last_scrub_stamp 2023-04-30T18:55:31.335752+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=31, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:20:37.986 INFO:tasks.ceph:pgid 8.6 last_scrub_stamp 2023-04-30T18:55:31.335752+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=31, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.986 INFO:tasks.ceph:pgid 9.7 last_scrub_stamp 2023-04-30T18:55:55.391997+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=55, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.986 INFO:tasks.ceph:pgid 10.4 last_scrub_stamp 2023-04-30T18:56:43.320182+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=56, tm_sec=43, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:20:38.015 INFO:tasks.ceph:pgid 9.2 last_scrub_stamp 2023-04-30T18:55:55.391997+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=55, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.015 INFO:tasks.ceph:pgid 10.1 last_scrub_stamp 2023-04-30T18:56:43.320182+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=56, tm_sec=43, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.015 INFO:tasks.ceph:pgid 11.0 last_scrub_stamp 2023-04-30T18:57:00.371165+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=57, tm_sec=0, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:20:38.044 INFO:tasks.ceph:pgid 7.5 last_scrub_stamp 2023-04-30T18:55:17.299077+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=17, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.044 INFO:tasks.ceph:pgid 6.4 last_scrub_stamp 2023-04-30T18:55:02.438577+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=2, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.044 INFO:tasks.ceph:pgid 5.7 last_scrub_stamp 2023-04-30T18:54:56.415471+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=54, tm_sec=56, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:20:38.073 INFO:tasks.ceph:pgid 15.10 last_scrub_stamp 2023-04-30T19:07:46.490515+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=7, tm_sec=46, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.073 INFO:tasks.ceph:pgid 14.11 last_scrub_stamp 2023-04-30T19:05:30.431322+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=5, tm_sec=30, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.073 INFO:tasks.ceph:pgid 13.12 last_scrub_stamp 2023-04-30T18:57:33.358860+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=57, tm_sec=33, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:21:17.308 INFO:tasks.ceph.mgr.x:Stopped , 2023-04-30T19:21:17.308 INFO:teuthology.misc:Shutting down mon daemons... , 2023-04-30T19:21:17.309 DEBUG:tasks.ceph.mon.a:waiting for process to exit Ignoring malparsed backtrace: next(self.gen) , File "/home/teuthworker/src/git.ceph.com_teuthology_69caab8ec6f44c61c991ae693e8e7b8cd859a896/teuthology/task/internal/__init__.py", line 420, in archive , fetch_binaries_for_coredumps(path, rem) Ignoring malparsed backtrace: ceph version 18.0.0-3633-g0ae30b6b (0ae30b6b0a0fdd9fa3bc438c826208438ac27aa2) reef (dev) , 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x120) [0x7f9b219d07ff] , 2: (ceph::register_assert_context(ceph::common::CephContext*)+0) [0x7f9b219d0a2a] Ignoring malparsed backtrace: instructed osd(s) 0 to deep-scrub , 2023-04-30T19:20:36.407 INFO:tasks.ceph:Scrubbing osd.1 , 2023-04-30T19:20:36.408 DEBUG:teuthology.orchestra.run.smithi165:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.1 config set osd_debug_deep_scrub_sleep 0 Ignoring malparsed backtrace: 2023-04-30T19:20:37.927 INFO:tasks.ceph:pgid 6.12 last_scrub_stamp 2023-04-30T18:55:02.438577+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=2, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.927 INFO:tasks.ceph:pgid 5.11 last_scrub_stamp 2023-04-30T18:54:56.415471+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=54, tm_sec=56, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.928 INFO:tasks.ceph:pgid 4.10 last_scrub_stamp 2023-04-30T18:54:50.398271+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=54, tm_sec=50, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:20:37.957 INFO:tasks.ceph:pgid 5.14 last_scrub_stamp 2023-04-30T18:54:56.415471+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=54, tm_sec=56, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.957 INFO:tasks.ceph:pgid 4.15 last_scrub_stamp 2023-04-30T18:54:50.398271+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=54, tm_sec=50, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.957 INFO:tasks.ceph:pgid 8.19 last_scrub_stamp 2023-04-30T18:55:31.335752+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=31, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:20:37.986 INFO:tasks.ceph:pgid 8.6 last_scrub_stamp 2023-04-30T18:55:31.335752+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=31, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.986 INFO:tasks.ceph:pgid 9.7 last_scrub_stamp 2023-04-30T18:55:55.391997+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=55, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.986 INFO:tasks.ceph:pgid 10.4 last_scrub_stamp 2023-04-30T18:56:43.320182+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=56, tm_sec=43, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:20:38.015 INFO:tasks.ceph:pgid 9.2 last_scrub_stamp 2023-04-30T18:55:55.391997+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=55, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.015 INFO:tasks.ceph:pgid 10.1 last_scrub_stamp 2023-04-30T18:56:43.320182+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=56, tm_sec=43, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.015 INFO:tasks.ceph:pgid 11.0 last_scrub_stamp 2023-04-30T18:57:00.371165+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=57, tm_sec=0, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:20:38.044 INFO:tasks.ceph:pgid 7.5 last_scrub_stamp 2023-04-30T18:55:17.299077+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=17, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.044 INFO:tasks.ceph:pgid 6.4 last_scrub_stamp 2023-04-30T18:55:02.438577+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=2, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.044 INFO:tasks.ceph:pgid 5.7 last_scrub_stamp 2023-04-30T18:54:56.415471+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=54, tm_sec=56, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:20:38.073 INFO:tasks.ceph:pgid 15.10 last_scrub_stamp 2023-04-30T19:07:46.490515+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=7, tm_sec=46, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.073 INFO:tasks.ceph:pgid 14.11 last_scrub_stamp 2023-04-30T19:05:30.431322+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=5, tm_sec=30, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.073 INFO:tasks.ceph:pgid 13.12 last_scrub_stamp 2023-04-30T18:57:33.358860+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=57, tm_sec=33, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:21:17.308 INFO:tasks.ceph.mgr.x:Stopped , 2023-04-30T19:21:17.308 INFO:teuthology.misc:Shutting down mon daemons... , 2023-04-30T19:21:17.309 DEBUG:tasks.ceph.mon.a:waiting for process to exit Ignoring malparsed backtrace: next(self.gen) , File "/home/teuthworker/src/git.ceph.com_teuthology_69caab8ec6f44c61c991ae693e8e7b8cd859a896/teuthology/task/internal/__init__.py", line 420, in archive , fetch_binaries_for_coredumps(path, rem) Ignoring malparsed backtrace: ceph version 18.0.0-3633-g0ae30b6b (0ae30b6b0a0fdd9fa3bc438c826208438ac27aa2) reef (dev) , 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x120) [0x7f9b219d07ff] , 2: (ceph::register_assert_context(ceph::common::CephContext*)+0) [0x7f9b219d0a2a] Ignoring malparsed backtrace: instructed osd(s) 0 to deep-scrub , 2023-04-30T19:20:36.407 INFO:tasks.ceph:Scrubbing osd.1 , 2023-04-30T19:20:36.408 DEBUG:teuthology.orchestra.run.smithi165:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.1 config set osd_debug_deep_scrub_sleep 0 Ignoring malparsed backtrace: 2023-04-30T19:20:37.927 INFO:tasks.ceph:pgid 6.12 last_scrub_stamp 2023-04-30T18:55:02.438577+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=2, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.927 INFO:tasks.ceph:pgid 5.11 last_scrub_stamp 2023-04-30T18:54:56.415471+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=54, tm_sec=56, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.928 INFO:tasks.ceph:pgid 4.10 last_scrub_stamp 2023-04-30T18:54:50.398271+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=54, tm_sec=50, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:20:37.957 INFO:tasks.ceph:pgid 5.14 last_scrub_stamp 2023-04-30T18:54:56.415471+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=54, tm_sec=56, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.957 INFO:tasks.ceph:pgid 4.15 last_scrub_stamp 2023-04-30T18:54:50.398271+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=54, tm_sec=50, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.957 INFO:tasks.ceph:pgid 8.19 last_scrub_stamp 2023-04-30T18:55:31.335752+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=31, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:20:37.986 INFO:tasks.ceph:pgid 8.6 last_scrub_stamp 2023-04-30T18:55:31.335752+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=31, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.986 INFO:tasks.ceph:pgid 9.7 last_scrub_stamp 2023-04-30T18:55:55.391997+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=55, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.986 INFO:tasks.ceph:pgid 10.4 last_scrub_stamp 2023-04-30T18:56:43.320182+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=56, tm_sec=43, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:20:38.015 INFO:tasks.ceph:pgid 9.2 last_scrub_stamp 2023-04-30T18:55:55.391997+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=55, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.015 INFO:tasks.ceph:pgid 10.1 last_scrub_stamp 2023-04-30T18:56:43.320182+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=56, tm_sec=43, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.015 INFO:tasks.ceph:pgid 11.0 last_scrub_stamp 2023-04-30T18:57:00.371165+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=57, tm_sec=0, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:20:38.044 INFO:tasks.ceph:pgid 7.5 last_scrub_stamp 2023-04-30T18:55:17.299077+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=17, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.044 INFO:tasks.ceph:pgid 6.4 last_scrub_stamp 2023-04-30T18:55:02.438577+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=2, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.044 INFO:tasks.ceph:pgid 5.7 last_scrub_stamp 2023-04-30T18:54:56.415471+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=54, tm_sec=56, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:20:38.073 INFO:tasks.ceph:pgid 15.10 last_scrub_stamp 2023-04-30T19:07:46.490515+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=7, tm_sec=46, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.073 INFO:tasks.ceph:pgid 14.11 last_scrub_stamp 2023-04-30T19:05:30.431322+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=5, tm_sec=30, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.073 INFO:tasks.ceph:pgid 13.12 last_scrub_stamp 2023-04-30T18:57:33.358860+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=57, tm_sec=33, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:21:17.308 INFO:tasks.ceph.mgr.x:Stopped , 2023-04-30T19:21:17.308 INFO:teuthology.misc:Shutting down mon daemons... , 2023-04-30T19:21:17.309 DEBUG:tasks.ceph.mon.a:waiting for process to exit Ignoring malparsed backtrace: next(self.gen) , File "/home/teuthworker/src/git.ceph.com_teuthology_69caab8ec6f44c61c991ae693e8e7b8cd859a896/teuthology/task/internal/__init__.py", line 420, in archive , fetch_binaries_for_coredumps(path, rem) Ignoring malparsed backtrace: ceph version 18.0.0-3633-g0ae30b6b (0ae30b6b0a0fdd9fa3bc438c826208438ac27aa2) reef (dev) , 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x120) [0x7f9b219d07ff] , 2: (ceph::register_assert_context(ceph::common::CephContext*)+0) [0x7f9b219d0a2a] Ignoring malparsed backtrace: instructed osd(s) 0 to deep-scrub , 2023-04-30T19:20:36.407 INFO:tasks.ceph:Scrubbing osd.1 , 2023-04-30T19:20:36.408 DEBUG:teuthology.orchestra.run.smithi165:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.1 config set osd_debug_deep_scrub_sleep 0 Ignoring malparsed backtrace: 2023-04-30T19:20:37.927 INFO:tasks.ceph:pgid 6.12 last_scrub_stamp 2023-04-30T18:55:02.438577+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=2, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.927 INFO:tasks.ceph:pgid 5.11 last_scrub_stamp 2023-04-30T18:54:56.415471+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=54, tm_sec=56, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.928 INFO:tasks.ceph:pgid 4.10 last_scrub_stamp 2023-04-30T18:54:50.398271+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=54, tm_sec=50, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:20:37.957 INFO:tasks.ceph:pgid 5.14 last_scrub_stamp 2023-04-30T18:54:56.415471+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=54, tm_sec=56, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.957 INFO:tasks.ceph:pgid 4.15 last_scrub_stamp 2023-04-30T18:54:50.398271+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=54, tm_sec=50, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.957 INFO:tasks.ceph:pgid 8.19 last_scrub_stamp 2023-04-30T18:55:31.335752+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=31, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:20:37.986 INFO:tasks.ceph:pgid 8.6 last_scrub_stamp 2023-04-30T18:55:31.335752+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=31, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.986 INFO:tasks.ceph:pgid 9.7 last_scrub_stamp 2023-04-30T18:55:55.391997+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=55, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:37.986 INFO:tasks.ceph:pgid 10.4 last_scrub_stamp 2023-04-30T18:56:43.320182+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=56, tm_sec=43, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:20:38.015 INFO:tasks.ceph:pgid 9.2 last_scrub_stamp 2023-04-30T18:55:55.391997+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=55, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.015 INFO:tasks.ceph:pgid 10.1 last_scrub_stamp 2023-04-30T18:56:43.320182+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=56, tm_sec=43, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.015 INFO:tasks.ceph:pgid 11.0 last_scrub_stamp 2023-04-30T18:57:00.371165+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=57, tm_sec=0, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:20:38.044 INFO:tasks.ceph:pgid 7.5 last_scrub_stamp 2023-04-30T18:55:17.299077+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=17, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.044 INFO:tasks.ceph:pgid 6.4 last_scrub_stamp 2023-04-30T18:55:02.438577+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=55, tm_sec=2, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.044 INFO:tasks.ceph:pgid 5.7 last_scrub_stamp 2023-04-30T18:54:56.415471+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=54, tm_sec=56, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:20:38.073 INFO:tasks.ceph:pgid 15.10 last_scrub_stamp 2023-04-30T19:07:46.490515+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=7, tm_sec=46, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.073 INFO:tasks.ceph:pgid 14.11 last_scrub_stamp 2023-04-30T19:05:30.431322+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=5, tm_sec=30, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) , 2023-04-30T19:20:38.073 INFO:tasks.ceph:pgid 13.12 last_scrub_stamp 2023-04-30T18:57:33.358860+0000 time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=18, tm_min=57, tm_sec=33, tm_wday=6, tm_yday=120, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=4, tm_mday=30, tm_hour=19, tm_min=20, tm_sec=34, tm_wday=6, tm_yday=120, tm_isdst=0) Ignoring malparsed backtrace: 2023-04-30T19:21:17.308 INFO:tasks.ceph.mgr.x:Stopped , 2023-04-30T19:21:17.308 INFO:teuthology.misc:Shutting down mon daemons... , 2023-04-30T19:21:17.309 DEBUG:tasks.ceph.mon.a:waiting for process to exit Ignoring malparsed backtrace: next(self.gen) , File "/home/teuthworker/src/git.ceph.com_teuthology_69caab8ec6f44c61c991ae693e8e7b8cd859a896/teuthology/task/internal/__init__.py", line 420, in archive , fetch_binaries_for_coredumps(path, rem) Found 9 distinct failure reasons Failure: Command failed (workunit test rbd/test_librbd_python.sh) on smithi027 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=135726148d97bd4e65d5b006464f156f7132999b TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 CRIMSON_COMPAT=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/test_librbd_python.sh' 1 jobs: ['7258775'] suites: ['clusters/fixed-1', 'crimson-rados/rbd/{centos_latest', 'crimson_qa_overrides', 'deploy/ceph', 'tasks/rbd_python_api_tests_old_format}'] Dead: b'2023-05-01T05:03:10.717 DEBUG:teuthology.exit:Finished running handlers' 2 jobs: ['7258762', '7258767'] suites intersection: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{default}', 'centos_latest', 'clusters/{fixed-2}', 'crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'crimson_qa_overrides', 'deploy/ceph', 'objectstore/bluestore', 'thrashers/default', 'thrashosds-health'] suites union: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{default}', 'centos_latest', 'clusters/{fixed-2}', 'crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'crimson_qa_overrides', 'deploy/ceph', 'objectstore/bluestore', 'thrashers/default', 'thrashosds-health', 'workloads/pool-snaps-few-objects}', 'workloads/small-objects-balanced}'] Failure: Command crashed: 'CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --localize-reads --max-ops 400000 --objects 1024 --max-in-flight 64 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 600 --op read 100 --op write 50 --op delete 50 --op snap_create 50 --op snap_remove 50 --op rollback 0 --op setattr 25 --op rmattr 25 --op copy_from 0 --op write_excl 50 --pool unique_pool_0' 1 jobs: ['7258769'] suites: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{default}', 'centos_latest', 'clusters/{fixed-2}', 'crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'crimson_qa_overrides', 'deploy/ceph', 'objectstore/bluestore', 'thrashers/default', 'thrashosds-health', 'workloads/small-objects-localized}'] Dead: b'2023-05-01T05:08:43.571 INFO:teuthology.orchestra.run.smithi132.stderr:2023-05-01T05:08:43.571+0000 7fb8d632d700 -1 WARNING: the following dangerous and experimental features are enabled: crimson' 1 jobs: ['7258770'] suites: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{default}', 'centos_latest', 'clusters/{fixed-2}', 'crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'crimson_qa_overrides', 'deploy/ceph', 'objectstore/bluestore', 'thrashers/default', 'thrashosds-health', 'workloads/small-objects}'] Failure: Command failed (workunit test rbd/test_lock_fence.sh) on smithi060 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=135726148d97bd4e65d5b006464f156f7132999b TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 CRIMSON_COMPAT=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/test_lock_fence.sh' 1 jobs: ['7258771'] suites: ['clusters/fixed-1', 'crimson-rados/rbd/{centos_latest', 'crimson_qa_overrides', 'deploy/ceph', 'tasks/rbd_lock_and_fence}'] Failure: Command crashed: 'CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --balance-reads --max-ops 4000 --objects 50 --max-in-flight 16 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 0 --op read 100 --op write 50 --op delete 50 --op snap_create 50 --op snap_remove 50 --op rollback 0 --op copy_from 0 --op write_excl 50 --pool unique_pool_0' 3 jobs: ['7258772', '7258776', '7258774'] suites intersection: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{default}', 'centos_latest', 'clusters/{fixed-2}', 'crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'crimson_qa_overrides', 'deploy/ceph', 'objectstore/bluestore', 'thrashers/default', 'thrashosds-health'] suites union: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{default}', 'centos_latest', 'clusters/{fixed-2}', 'crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'crimson_qa_overrides', 'deploy/ceph', 'objectstore/bluestore', 'thrashers/default', 'thrashosds-health', 'workloads/snaps-few-objects-balanced}', 'workloads/snaps-few-objects-localized}', 'workloads/snaps-few-objects}'] Failure: Command failed (workunit test cls/test_cls_rbd.sh) on smithi174 with status 139: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=135726148d97bd4e65d5b006464f156f7132999b TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 CRIMSON_COMPAT=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 1 jobs: ['7258768'] suites: ['clusters/fixed-1', 'crimson-rados/rbd/{centos_latest', 'crimson_qa_overrides', 'deploy/ceph', 'tasks/rbd_cls_tests}'] Timeout 3h running clone.client.0/qa/workunits/rados/test_pool_quota.sh 1 jobs: ['7258759'] suites: ['clusters/fixed-2', 'crimson-rados/basic/{centos_latest', 'crimson_qa_overrides', 'deploy/ceph', 'tasks/rados_api_tests}'] Failure: Command failed (workunit test rbd/crimson/test_crimson_librbd.sh) on smithi165 with status 134: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=135726148d97bd4e65d5b006464f156f7132999b TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_FEATURES=61 CRIMSON_COMPAT=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/crimson/test_crimson_librbd.sh' 1 jobs: ['7258760'] suites: ['clusters/fixed-1', 'crimson-rados/rbd/{centos_latest', 'crimson_qa_overrides', 'deploy/ceph', 'tasks/rbd_api_tests}']