Found 2 jobs Ignoring malparsed backtrace: ceph version 18.0.0-1760-g37e3bc63 (37e3bc63cee331f2aee7c1f18242b0444fc7d59c) reef (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7fcc12309eb0] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x1b0) [0x5555798640ca] Ignoring malparsed backtrace: 2023-01-15T13:14:57.832 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~50s , 2023-01-15T13:14:58.899 DEBUG:teuthology.orchestra.run.smithi067:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-01-15T13:14:59.243 INFO:teuthology.orchestra.run.smithi067.stdout: Ignoring malparsed backtrace: 2023-01-15T13:16:32.551 INFO:teuthology.orchestra.run.smithi067.stdout:{"epoch":76,"fsid":"c7653025-3fd0-4172-809a-8ad4a7a58e63","created":"2023-01-15T13:12:21.023479+0000","modified":"2023-01-15T13:14:42.078840+0000","last_up_change":"2023-01-15T13:14:27.013948+0000","last_in_change":"2023-01-15T13:12:22.273342+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-01-15T13:12:28.806998+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-01-15T13:12:30.639872+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-01-15T13:12:42.973295+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":44,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"8ed9b95f-bb39-406b-a1e1-4c1c9ddc9743","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":15,"down_at":66,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6801","nonce":102024}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6800","nonce":102024}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6805","nonce":102024}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6804","nonce":102024}]},"public_addr":"172.21.15.67:6801/102024","cluster_addr":"172.21.15.67:6800/102024","heartbeat_back_addr":"172.21.15.67:6805/102024","heartbeat_front_addr":"172.21.15.67:6804/102024","state":["exists"]},{"osd":1,"uuid":"a5f994bd-e945-4914-972c-bcbb43ad2f07","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6803","nonce":102051}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6802","nonce":102051}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6807","nonce":102051}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6806","nonce":102051}]},"public_addr":"172.21.15.67:6803/102051","cluster_addr":"172.21.15.67:6802/102051","heartbeat_back_addr":"172.21.15.67:6807/102051","heartbeat_front_addr":"172.21.15.67:6806/102051","state":["exists","up"]},{"osd":2,"uuid":"5062ee8e-2aab-4cfb-906e-7d195405ec2e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6800","nonce":101651}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6801","nonce":101651}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6804","nonce":101651}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6805","nonce":101651}]},"public_addr":"172.21.15.122:6800/101651","cluster_addr":"172.21.15.122:6801/101651","heartbeat_back_addr":"172.21.15.122:6804/101651","heartbeat_front_addr":"172.21.15.122:6805/101651","state":["exists","up"]},{"osd":3,"uuid":"d7a1d6e6-497a-4fa1-9b9b-e4abe2127a19","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6803","nonce":101678}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6802","nonce":101678}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6807","nonce":101678}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6806","nonce":101678}]},"public_addr":"172.21.15.122:6803/101678","cluster_addr":"172.21.15.122:6802/101678","heartbeat_back_addr":"172.21.15.122:6807/101678","heartbeat_front_addr":"172.21.15.122:6806/101678","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-01-15T13:14:27.013948+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-01-15T13:16:35.566 DEBUG:teuthology.orchestra.run.smithi067:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-01-15T13:16:35.911 INFO:teuthology.orchestra.run.smithi067.stdout: Ignoring malparsed backtrace: 2023-01-15T13:18:08.814 INFO:teuthology.orchestra.run.smithi067.stdout:{"epoch":76,"fsid":"c7653025-3fd0-4172-809a-8ad4a7a58e63","created":"2023-01-15T13:12:21.023479+0000","modified":"2023-01-15T13:14:42.078840+0000","last_up_change":"2023-01-15T13:14:27.013948+0000","last_in_change":"2023-01-15T13:12:22.273342+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-01-15T13:12:28.806998+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-01-15T13:12:30.639872+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-01-15T13:12:42.973295+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":44,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"8ed9b95f-bb39-406b-a1e1-4c1c9ddc9743","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":15,"down_at":66,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6801","nonce":102024}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6800","nonce":102024}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6805","nonce":102024}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6804","nonce":102024}]},"public_addr":"172.21.15.67:6801/102024","cluster_addr":"172.21.15.67:6800/102024","heartbeat_back_addr":"172.21.15.67:6805/102024","heartbeat_front_addr":"172.21.15.67:6804/102024","state":["exists"]},{"osd":1,"uuid":"a5f994bd-e945-4914-972c-bcbb43ad2f07","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6803","nonce":102051}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6802","nonce":102051}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6807","nonce":102051}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6806","nonce":102051}]},"public_addr":"172.21.15.67:6803/102051","cluster_addr":"172.21.15.67:6802/102051","heartbeat_back_addr":"172.21.15.67:6807/102051","heartbeat_front_addr":"172.21.15.67:6806/102051","state":["exists","up"]},{"osd":2,"uuid":"5062ee8e-2aab-4cfb-906e-7d195405ec2e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6800","nonce":101651}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6801","nonce":101651}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6804","nonce":101651}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6805","nonce":101651}]},"public_addr":"172.21.15.122:6800/101651","cluster_addr":"172.21.15.122:6801/101651","heartbeat_back_addr":"172.21.15.122:6804/101651","heartbeat_front_addr":"172.21.15.122:6805/101651","state":["exists","up"]},{"osd":3,"uuid":"d7a1d6e6-497a-4fa1-9b9b-e4abe2127a19","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6803","nonce":101678}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6802","nonce":101678}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6807","nonce":101678}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6806","nonce":101678}]},"public_addr":"172.21.15.122:6803/101678","cluster_addr":"172.21.15.122:6802/101678","heartbeat_back_addr":"172.21.15.122:6807/101678","heartbeat_front_addr":"172.21.15.122:6806/101678","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-01-15T13:14:27.013948+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-01-15T13:18:11.167 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~243s , 2023-01-15T13:18:11.829 DEBUG:teuthology.orchestra.run.smithi067:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json Ignoring malparsed backtrace: raise CommandCrashedError(command=self.command) , teuthology.exceptions.CommandCrashedError: Command crashed: 'CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --balance-reads --max-ops 400000 --objects 1024 --max-in-flight 64 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 600 --op read 100 --op write 50 --op delete 50 --op snap_create 50 --op snap_remove 0 --op rollback 0 --op setattr 25 --op rmattr 25 --op copy_from 0 --op write_excl 50 --pool unique_pool_0' , Ignoring malparsed backtrace: File "src/gevent/greenlet.py", line 829, in gevent._gevent_cgreenlet.Greenlet.get , File "src/gevent/greenlet.py", line 371, in gevent._gevent_cgreenlet.Greenlet._raise_exception , File "/home/teuthworker/src/git.ceph.com_teuthology_19d18db866afdc31fd6586f558fc29b95a87ccfb/virtualenv/lib/python3.8/site-packages/gevent/_compat.py", line 65, in reraise Ignoring malparsed backtrace: 2023-01-15T13:23:15.091 DEBUG:teuthology.misc:Transferring archived files from smithi122:/var/lib/ceph/crash to /home/teuthworker/archive/matan-2023-01-15_12:40:04-crimson-rados-wip-matanb-crimson-testing-1.11-balance-distro-crimson-smithi/7142795/remote/smithi122/crash , 2023-01-15T13:23:15.092 DEBUG:teuthology.orchestra.run.smithi122:> sudo tar cz -f - -C /var/lib/ceph/crash -- . , 2023-01-15T13:23:15.131 INFO:tasks.ceph:Compressing logs... Ignoring malparsed backtrace: 2023-01-15T13:23:20.704 INFO:teuthology.orchestra.run.smithi122.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 47 +100us[ +101us] +/- 59ms , 2023-01-15T13:23:20.704 INFO:teuthology.orchestra.run.smithi122.stdout:^? hv04.front.sepia.ceph.com 0 8 0 - +0ns[ +0ns] +/- 0ns , 2023-01-15T13:23:20.705 INFO:teuthology.orchestra.run.smithi067.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample Ignoring malparsed backtrace: self.gen.throw(type, value, traceback) , File "/home/teuthworker/src/git.ceph.com_teuthology_19d18db866afdc31fd6586f558fc29b95a87ccfb/teuthology/task/internal/__init__.py", line 421, in archive , fetch_binaries_for_coredumps(path, rem) Ignoring malparsed backtrace: ceph version 18.0.0-1760-g37e3bc63 (37e3bc63cee331f2aee7c1f18242b0444fc7d59c) reef (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7fcc12309eb0] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x1b0) [0x5555798640ca] Ignoring malparsed backtrace: 2023-01-15T13:14:57.832 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~50s , 2023-01-15T13:14:58.899 DEBUG:teuthology.orchestra.run.smithi067:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-01-15T13:14:59.243 INFO:teuthology.orchestra.run.smithi067.stdout: Ignoring malparsed backtrace: 2023-01-15T13:16:32.551 INFO:teuthology.orchestra.run.smithi067.stdout:{"epoch":76,"fsid":"c7653025-3fd0-4172-809a-8ad4a7a58e63","created":"2023-01-15T13:12:21.023479+0000","modified":"2023-01-15T13:14:42.078840+0000","last_up_change":"2023-01-15T13:14:27.013948+0000","last_in_change":"2023-01-15T13:12:22.273342+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-01-15T13:12:28.806998+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-01-15T13:12:30.639872+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-01-15T13:12:42.973295+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":44,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"8ed9b95f-bb39-406b-a1e1-4c1c9ddc9743","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":15,"down_at":66,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6801","nonce":102024}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6800","nonce":102024}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6805","nonce":102024}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6804","nonce":102024}]},"public_addr":"172.21.15.67:6801/102024","cluster_addr":"172.21.15.67:6800/102024","heartbeat_back_addr":"172.21.15.67:6805/102024","heartbeat_front_addr":"172.21.15.67:6804/102024","state":["exists"]},{"osd":1,"uuid":"a5f994bd-e945-4914-972c-bcbb43ad2f07","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6803","nonce":102051}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6802","nonce":102051}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6807","nonce":102051}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6806","nonce":102051}]},"public_addr":"172.21.15.67:6803/102051","cluster_addr":"172.21.15.67:6802/102051","heartbeat_back_addr":"172.21.15.67:6807/102051","heartbeat_front_addr":"172.21.15.67:6806/102051","state":["exists","up"]},{"osd":2,"uuid":"5062ee8e-2aab-4cfb-906e-7d195405ec2e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6800","nonce":101651}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6801","nonce":101651}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6804","nonce":101651}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6805","nonce":101651}]},"public_addr":"172.21.15.122:6800/101651","cluster_addr":"172.21.15.122:6801/101651","heartbeat_back_addr":"172.21.15.122:6804/101651","heartbeat_front_addr":"172.21.15.122:6805/101651","state":["exists","up"]},{"osd":3,"uuid":"d7a1d6e6-497a-4fa1-9b9b-e4abe2127a19","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6803","nonce":101678}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6802","nonce":101678}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6807","nonce":101678}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6806","nonce":101678}]},"public_addr":"172.21.15.122:6803/101678","cluster_addr":"172.21.15.122:6802/101678","heartbeat_back_addr":"172.21.15.122:6807/101678","heartbeat_front_addr":"172.21.15.122:6806/101678","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-01-15T13:14:27.013948+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-01-15T13:16:35.566 DEBUG:teuthology.orchestra.run.smithi067:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-01-15T13:16:35.911 INFO:teuthology.orchestra.run.smithi067.stdout: Ignoring malparsed backtrace: 2023-01-15T13:18:08.814 INFO:teuthology.orchestra.run.smithi067.stdout:{"epoch":76,"fsid":"c7653025-3fd0-4172-809a-8ad4a7a58e63","created":"2023-01-15T13:12:21.023479+0000","modified":"2023-01-15T13:14:42.078840+0000","last_up_change":"2023-01-15T13:14:27.013948+0000","last_in_change":"2023-01-15T13:12:22.273342+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-01-15T13:12:28.806998+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-01-15T13:12:30.639872+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-01-15T13:12:42.973295+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":44,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"8ed9b95f-bb39-406b-a1e1-4c1c9ddc9743","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":15,"down_at":66,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6801","nonce":102024}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6800","nonce":102024}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6805","nonce":102024}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6804","nonce":102024}]},"public_addr":"172.21.15.67:6801/102024","cluster_addr":"172.21.15.67:6800/102024","heartbeat_back_addr":"172.21.15.67:6805/102024","heartbeat_front_addr":"172.21.15.67:6804/102024","state":["exists"]},{"osd":1,"uuid":"a5f994bd-e945-4914-972c-bcbb43ad2f07","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6803","nonce":102051}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6802","nonce":102051}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6807","nonce":102051}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6806","nonce":102051}]},"public_addr":"172.21.15.67:6803/102051","cluster_addr":"172.21.15.67:6802/102051","heartbeat_back_addr":"172.21.15.67:6807/102051","heartbeat_front_addr":"172.21.15.67:6806/102051","state":["exists","up"]},{"osd":2,"uuid":"5062ee8e-2aab-4cfb-906e-7d195405ec2e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6800","nonce":101651}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6801","nonce":101651}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6804","nonce":101651}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6805","nonce":101651}]},"public_addr":"172.21.15.122:6800/101651","cluster_addr":"172.21.15.122:6801/101651","heartbeat_back_addr":"172.21.15.122:6804/101651","heartbeat_front_addr":"172.21.15.122:6805/101651","state":["exists","up"]},{"osd":3,"uuid":"d7a1d6e6-497a-4fa1-9b9b-e4abe2127a19","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6803","nonce":101678}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6802","nonce":101678}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6807","nonce":101678}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6806","nonce":101678}]},"public_addr":"172.21.15.122:6803/101678","cluster_addr":"172.21.15.122:6802/101678","heartbeat_back_addr":"172.21.15.122:6807/101678","heartbeat_front_addr":"172.21.15.122:6806/101678","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-01-15T13:14:27.013948+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-01-15T13:18:11.167 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~243s , 2023-01-15T13:18:11.829 DEBUG:teuthology.orchestra.run.smithi067:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json Ignoring malparsed backtrace: raise CommandCrashedError(command=self.command) , teuthology.exceptions.CommandCrashedError: Command crashed: 'CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --balance-reads --max-ops 400000 --objects 1024 --max-in-flight 64 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 600 --op read 100 --op write 50 --op delete 50 --op snap_create 50 --op snap_remove 0 --op rollback 0 --op setattr 25 --op rmattr 25 --op copy_from 0 --op write_excl 50 --pool unique_pool_0' , Ignoring malparsed backtrace: File "src/gevent/greenlet.py", line 829, in gevent._gevent_cgreenlet.Greenlet.get , File "src/gevent/greenlet.py", line 371, in gevent._gevent_cgreenlet.Greenlet._raise_exception , File "/home/teuthworker/src/git.ceph.com_teuthology_19d18db866afdc31fd6586f558fc29b95a87ccfb/virtualenv/lib/python3.8/site-packages/gevent/_compat.py", line 65, in reraise Ignoring malparsed backtrace: 2023-01-15T13:23:15.091 DEBUG:teuthology.misc:Transferring archived files from smithi122:/var/lib/ceph/crash to /home/teuthworker/archive/matan-2023-01-15_12:40:04-crimson-rados-wip-matanb-crimson-testing-1.11-balance-distro-crimson-smithi/7142795/remote/smithi122/crash , 2023-01-15T13:23:15.092 DEBUG:teuthology.orchestra.run.smithi122:> sudo tar cz -f - -C /var/lib/ceph/crash -- . , 2023-01-15T13:23:15.131 INFO:tasks.ceph:Compressing logs... Ignoring malparsed backtrace: 2023-01-15T13:23:20.704 INFO:teuthology.orchestra.run.smithi122.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 47 +100us[ +101us] +/- 59ms , 2023-01-15T13:23:20.704 INFO:teuthology.orchestra.run.smithi122.stdout:^? hv04.front.sepia.ceph.com 0 8 0 - +0ns[ +0ns] +/- 0ns , 2023-01-15T13:23:20.705 INFO:teuthology.orchestra.run.smithi067.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample Ignoring malparsed backtrace: self.gen.throw(type, value, traceback) , File "/home/teuthworker/src/git.ceph.com_teuthology_19d18db866afdc31fd6586f558fc29b95a87ccfb/teuthology/task/internal/__init__.py", line 421, in archive , fetch_binaries_for_coredumps(path, rem) Ignoring malparsed backtrace: ceph version 18.0.0-1760-g37e3bc63 (37e3bc63cee331f2aee7c1f18242b0444fc7d59c) reef (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7fcc12309eb0] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x1b0) [0x5555798640ca] Ignoring malparsed backtrace: 2023-01-15T13:14:57.832 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~50s , 2023-01-15T13:14:58.899 DEBUG:teuthology.orchestra.run.smithi067:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-01-15T13:14:59.243 INFO:teuthology.orchestra.run.smithi067.stdout: Ignoring malparsed backtrace: 2023-01-15T13:16:32.551 INFO:teuthology.orchestra.run.smithi067.stdout:{"epoch":76,"fsid":"c7653025-3fd0-4172-809a-8ad4a7a58e63","created":"2023-01-15T13:12:21.023479+0000","modified":"2023-01-15T13:14:42.078840+0000","last_up_change":"2023-01-15T13:14:27.013948+0000","last_in_change":"2023-01-15T13:12:22.273342+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-01-15T13:12:28.806998+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-01-15T13:12:30.639872+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-01-15T13:12:42.973295+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":44,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"8ed9b95f-bb39-406b-a1e1-4c1c9ddc9743","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":15,"down_at":66,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6801","nonce":102024}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6800","nonce":102024}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6805","nonce":102024}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6804","nonce":102024}]},"public_addr":"172.21.15.67:6801/102024","cluster_addr":"172.21.15.67:6800/102024","heartbeat_back_addr":"172.21.15.67:6805/102024","heartbeat_front_addr":"172.21.15.67:6804/102024","state":["exists"]},{"osd":1,"uuid":"a5f994bd-e945-4914-972c-bcbb43ad2f07","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6803","nonce":102051}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6802","nonce":102051}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6807","nonce":102051}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6806","nonce":102051}]},"public_addr":"172.21.15.67:6803/102051","cluster_addr":"172.21.15.67:6802/102051","heartbeat_back_addr":"172.21.15.67:6807/102051","heartbeat_front_addr":"172.21.15.67:6806/102051","state":["exists","up"]},{"osd":2,"uuid":"5062ee8e-2aab-4cfb-906e-7d195405ec2e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6800","nonce":101651}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6801","nonce":101651}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6804","nonce":101651}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6805","nonce":101651}]},"public_addr":"172.21.15.122:6800/101651","cluster_addr":"172.21.15.122:6801/101651","heartbeat_back_addr":"172.21.15.122:6804/101651","heartbeat_front_addr":"172.21.15.122:6805/101651","state":["exists","up"]},{"osd":3,"uuid":"d7a1d6e6-497a-4fa1-9b9b-e4abe2127a19","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6803","nonce":101678}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6802","nonce":101678}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6807","nonce":101678}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6806","nonce":101678}]},"public_addr":"172.21.15.122:6803/101678","cluster_addr":"172.21.15.122:6802/101678","heartbeat_back_addr":"172.21.15.122:6807/101678","heartbeat_front_addr":"172.21.15.122:6806/101678","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-01-15T13:14:27.013948+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-01-15T13:16:35.566 DEBUG:teuthology.orchestra.run.smithi067:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-01-15T13:16:35.911 INFO:teuthology.orchestra.run.smithi067.stdout: Ignoring malparsed backtrace: 2023-01-15T13:18:08.814 INFO:teuthology.orchestra.run.smithi067.stdout:{"epoch":76,"fsid":"c7653025-3fd0-4172-809a-8ad4a7a58e63","created":"2023-01-15T13:12:21.023479+0000","modified":"2023-01-15T13:14:42.078840+0000","last_up_change":"2023-01-15T13:14:27.013948+0000","last_in_change":"2023-01-15T13:12:22.273342+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-01-15T13:12:28.806998+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-01-15T13:12:30.639872+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-01-15T13:12:42.973295+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":44,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"8ed9b95f-bb39-406b-a1e1-4c1c9ddc9743","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":15,"down_at":66,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6801","nonce":102024}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6800","nonce":102024}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6805","nonce":102024}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6804","nonce":102024}]},"public_addr":"172.21.15.67:6801/102024","cluster_addr":"172.21.15.67:6800/102024","heartbeat_back_addr":"172.21.15.67:6805/102024","heartbeat_front_addr":"172.21.15.67:6804/102024","state":["exists"]},{"osd":1,"uuid":"a5f994bd-e945-4914-972c-bcbb43ad2f07","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6803","nonce":102051}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6802","nonce":102051}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6807","nonce":102051}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6806","nonce":102051}]},"public_addr":"172.21.15.67:6803/102051","cluster_addr":"172.21.15.67:6802/102051","heartbeat_back_addr":"172.21.15.67:6807/102051","heartbeat_front_addr":"172.21.15.67:6806/102051","state":["exists","up"]},{"osd":2,"uuid":"5062ee8e-2aab-4cfb-906e-7d195405ec2e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6800","nonce":101651}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6801","nonce":101651}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6804","nonce":101651}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6805","nonce":101651}]},"public_addr":"172.21.15.122:6800/101651","cluster_addr":"172.21.15.122:6801/101651","heartbeat_back_addr":"172.21.15.122:6804/101651","heartbeat_front_addr":"172.21.15.122:6805/101651","state":["exists","up"]},{"osd":3,"uuid":"d7a1d6e6-497a-4fa1-9b9b-e4abe2127a19","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6803","nonce":101678}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6802","nonce":101678}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6807","nonce":101678}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.122:6806","nonce":101678}]},"public_addr":"172.21.15.122:6803/101678","cluster_addr":"172.21.15.122:6802/101678","heartbeat_back_addr":"172.21.15.122:6807/101678","heartbeat_front_addr":"172.21.15.122:6806/101678","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-01-15T13:14:27.013948+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-01-15T13:18:11.167 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~243s , 2023-01-15T13:18:11.829 DEBUG:teuthology.orchestra.run.smithi067:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json Ignoring malparsed backtrace: raise CommandCrashedError(command=self.command) , teuthology.exceptions.CommandCrashedError: Command crashed: 'CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --balance-reads --max-ops 400000 --objects 1024 --max-in-flight 64 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 600 --op read 100 --op write 50 --op delete 50 --op snap_create 50 --op snap_remove 0 --op rollback 0 --op setattr 25 --op rmattr 25 --op copy_from 0 --op write_excl 50 --pool unique_pool_0' , Ignoring malparsed backtrace: File "src/gevent/greenlet.py", line 829, in gevent._gevent_cgreenlet.Greenlet.get , File "src/gevent/greenlet.py", line 371, in gevent._gevent_cgreenlet.Greenlet._raise_exception , File "/home/teuthworker/src/git.ceph.com_teuthology_19d18db866afdc31fd6586f558fc29b95a87ccfb/virtualenv/lib/python3.8/site-packages/gevent/_compat.py", line 65, in reraise Ignoring malparsed backtrace: 2023-01-15T13:23:15.091 DEBUG:teuthology.misc:Transferring archived files from smithi122:/var/lib/ceph/crash to /home/teuthworker/archive/matan-2023-01-15_12:40:04-crimson-rados-wip-matanb-crimson-testing-1.11-balance-distro-crimson-smithi/7142795/remote/smithi122/crash , 2023-01-15T13:23:15.092 DEBUG:teuthology.orchestra.run.smithi122:> sudo tar cz -f - -C /var/lib/ceph/crash -- . , 2023-01-15T13:23:15.131 INFO:tasks.ceph:Compressing logs... Ignoring malparsed backtrace: 2023-01-15T13:23:20.704 INFO:teuthology.orchestra.run.smithi122.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 47 +100us[ +101us] +/- 59ms , 2023-01-15T13:23:20.704 INFO:teuthology.orchestra.run.smithi122.stdout:^? hv04.front.sepia.ceph.com 0 8 0 - +0ns[ +0ns] +/- 0ns , 2023-01-15T13:23:20.705 INFO:teuthology.orchestra.run.smithi067.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample Ignoring malparsed backtrace: self.gen.throw(type, value, traceback) , File "/home/teuthworker/src/git.ceph.com_teuthology_19d18db866afdc31fd6586f558fc29b95a87ccfb/teuthology/task/internal/__init__.py", line 421, in archive , fetch_binaries_for_coredumps(path, rem) Ignoring malparsed backtrace: ceph version 18.0.0-1760-g37e3bc63 (37e3bc63cee331f2aee7c1f18242b0444fc7d59c) reef (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7f852eac7eb0] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x1b0) [0x562e961b00ca] Ignoring malparsed backtrace: 2023-01-15T13:10:15.970 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-01-15T13:10:16.189 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~55s , 2023-01-15T13:10:16.299 INFO:teuthology.orchestra.run.smithi090.stdout: Ignoring malparsed backtrace: 2023-01-15T13:11:51.457 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-01-15T13:11:51.578 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~150s , 2023-01-15T13:11:51.784 INFO:teuthology.orchestra.run.smithi090.stdout: Ignoring malparsed backtrace: 2023-01-15T13:13:25.836 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~245s , 2023-01-15T13:13:27.495 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-01-15T13:13:27.824 INFO:teuthology.orchestra.run.smithi090.stdout: Ignoring malparsed backtrace: File "/usr/lib/python3.8/contextlib.py", line 131, in __exit__ , self.gen.throw(type, value, traceback) , File "/home/teuthworker/src/github.com_Matan-B_ceph_20a5a76a31da5350bb2279f1355de07f4356a41a/qa/tasks/thrashosds.py", line 215, in task Ignoring malparsed backtrace: proc.wait() , File "/home/teuthworker/src/git.ceph.com_teuthology_19d18db866afdc31fd6586f558fc29b95a87ccfb/teuthology/orchestra/run.py", line 161, in wait , self._raise_for_status() Ignoring malparsed backtrace: 2023-01-15T13:18:28.318 DEBUG:teuthology.misc:Transferring archived files from smithi141:/var/log/ceph to /home/teuthworker/archive/matan-2023-01-15_12:40:04-crimson-rados-wip-matanb-crimson-testing-1.11-balance-distro-crimson-smithi/7142794/remote/smithi141/log , 2023-01-15T13:18:28.319 DEBUG:teuthology.orchestra.run.smithi141:> sudo tar cz -f - -C /var/log/ceph -- . , 2023-01-15T13:18:29.087 ERROR:teuthology.run_tasks:Manager failed: ceph Ignoring malparsed backtrace: 2023-01-15T13:18:29.250 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab , 2023-01-15T13:18:29.259 INFO:teuthology.task.ansible:Skipping ansible cleanup... , 2023-01-15T13:18:29.260 DEBUG:teuthology.run_tasks:Unwinding manager selinux Ignoring malparsed backtrace: r = self._runner(client=self.ssh, name=self.shortname, **kwargs) , File "/home/teuthworker/src/git.ceph.com_teuthology_19d18db866afdc31fd6586f558fc29b95a87ccfb/teuthology/orchestra/run.py", line 455, in run , r.wait() Found 1 distinct failure reasons Failure: Command crashed: 'CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --balance-reads --max-ops 400000 --objects 1024 --max-in-flight 64 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 600 --op read 100 --op write 50 --op delete 50 --op snap_create 50 --op snap_remove 0 --op rollback 0 --op setattr 25 --op rmattr 25 --op copy_from 0 --op write_excl 50 --pool unique_pool_0' 2 jobs: ['7142795', '7142794'] suites intersection: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{default}', 'centos_latest', 'clusters/{fixed-2}', 'crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'crimson_qa_overrides', 'deploy/ceph', 'objectstore/bluestore', 'thrashers/default', 'thrashosds-health', 'workloads/small-objects-balanced}'] suites union: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{default}', 'centos_latest', 'clusters/{fixed-2}', 'crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'crimson_qa_overrides', 'deploy/ceph', 'objectstore/bluestore', 'thrashers/default', 'thrashosds-health', 'workloads/small-objects-balanced}']