Found 60 jobs Ignoring malparsed backtrace: ceph version 18.0.0-4330-g9b3f3761 (9b3f3761ef331c3cedab822629c5de78b7d54292) reef (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7f07f68f8ed4] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x1b0) [0x5570926960ca] Ignoring malparsed backtrace: 2023-06-08T07:31:38.120 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":724,"fsid":"87dea55e-f8c6-48e7-a5de-497a76d690ca","created":"2023-06-08T07:19:03.434687+0000","modified":"2023-06-08T07:31:37.763740+0000","last_up_change":"2023-06-08T07:31:23.586612+0000","last_in_change":"2023-06-08T07:19:05.746653+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-06-08T07:19:11.856148+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-06-08T07:19:14.054099+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-06-08T07:19:21.041786+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"716","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":693,"snap_epoch":716,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"bc37cf1c-3e05-46a6-8dbc-b3e1a597b390","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":16,"down_at":711,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6800","nonce":2219628572}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6801","nonce":2219628572}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2219628572}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6805","nonce":2219628572}]},"public_addr":"172.21.15.99:6800/2219628572","cluster_addr":"172.21.15.99:6801/2219628572","heartbeat_back_addr":"172.21.15.99:6804/2219628572","heartbeat_front_addr":"172.21.15.99:6805/2219628572","state":["exists"]},{"osd":1,"uuid":"2628ebee-943b-4c37-b142-f4c7ab979a56","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":786871654}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6803","nonce":786871654}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6807","nonce":786871654}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":786871654}]},"public_addr":"172.21.15.99:6802/786871654","cluster_addr":"172.21.15.99:6803/786871654","heartbeat_back_addr":"172.21.15.99:6807/786871654","heartbeat_front_addr":"172.21.15.99:6806/786871654","state":["exists","up"]},{"osd":2,"uuid":"de6b6a32-e5a8-478f-8b75-94aebfb2b717","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6803","nonce":3055208471}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6802","nonce":3055208471}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6807","nonce":3055208471}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6808","nonce":3055208471}]},"public_addr":"172.21.15.162:6803/3055208471","cluster_addr":"172.21.15.162:6802/3055208471","heartbeat_back_addr":"172.21.15.162:6807/3055208471","heartbeat_front_addr":"172.21.15.162:6808/3055208471","state":["exists","up"]},{"osd":3,"uuid":"02b56dc3-6c99-49d7-b3d6-d5253d66b5c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6805","nonce":2372637129}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6804","nonce":2372637129}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6806","nonce":2372637129}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6809","nonce":2372637129}]},"public_addr":"172.21.15.162:6805/2372637129","cluster_addr":"172.21.15.162:6804/2372637129","heartbeat_back_addr":"172.21.15.162:6806/2372637129","heartbeat_front_addr":"172.21.15.162:6809/2372637129","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-06-08T07:31:23.586612+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":1},{"begin":8,"length":3},{"begin":14,"length":1},{"begin":16,"length":2},{"begin":20,"length":1},{"begin":22,"length":4},{"begin":27,"length":1},{"begin":31,"length":2},{"begin":36,"length":3},{"begin":40,"length":1},{"begin":42,"length":2},{"begin":45,"length":1},{"begin":49,"length":1},{"begin":51,"length":2},{"begin":55,"length":1},{"begin":57,"length":3},{"begin":61,"length":2},{"begin":64,"length":1},{"begin":68,"length":4},{"begin":75,"length":1},{"begin":77,"length":4},{"begin":82,"length":2},{"begin":86,"length":1},{"begin":89,"length":1},{"begin":91,"length":2},{"begin":94,"length":6},{"begin":101,"length":3},{"begin":106,"length":1},{"begin":108,"length":1},{"begin":111,"length":2},{"begin":114,"length":3},{"begin":118,"length":2},{"begin":121,"length":2},{"begin":124,"length":1},{"begin":135,"length":2},{"begin":138,"length":2},{"begin":142,"length":1},{"begin":145,"length":2},{"begin":148,"length":1},{"begin":150,"length":3},{"begin":160,"length":1},{"begin":162,"length":1},{"begin":165,"length":1},{"begin":169,"length":1},{"begin":171,"length":1},{"begin":173,"length":1},{"begin":176,"length":1},{"begin":178,"length":3},{"begin":184,"length":2},{"begin":188,"length":1},{"begin":190,"length":1},{"begin":192,"length":1},{"begin":194,"length":2},{"begin":198,"length":2},{"begin":202,"length":3},{"begin":208,"length":4},{"begin":215,"length":1},{"begin":218,"length":4},{"begin":226,"length":1},{"begin":228,"length":3},{"begin":233,"length":5},{"begin":240,"length":1},{"begin":243,"length":1},{"begin":246,"length":1},{"begin":248,"length":2},{"begin":251,"length":1},{"begin":253,"length":1},{"begin":256,"length":3},{"begin":263,"length":1},{"begin":266,"length":3},{"begin":270,"length":2},{"begin":274,"length":2},{"begin":277,"length":1},{"begin":280,"length":1},{"begin":284,"length":3},{"begin":288,"length":2},{"begin":291,"length":2},{"begin":295,"length":1},{"begin":297,"length":2},{"begin":303,"length":3},{"begin":307,"length":1},{"begin":309,"length":3},{"begin":313,"length":1},{"begin":316,"length":1},{"begin":319,"length":2},{"begin":323,"length":1},{"begin":326,"length":2},{"begin":329,"length":7},{"begin":338,"length":1},{"begin":342,"length":1},{"begin":345,"length":2},{"begin":348,"length":3},{"begin":353,"length":1},{"begin":357,"length":4},{"begin":363,"length":2},{"begin":369,"length":1},{"begin":373,"length":3},{"begin":377,"length":2},{"begin":380,"length":1},{"begin":382,"length":1},{"begin":385,"length":4},{"begin":393,"length":1},{"begin":396,"length":1},{"begin":401,"length":1},{"begin":403,"length":1},{"begin":405,"length":1},{"begin":407,"length":1},{"begin":409,"length":1},{"begin":411,"length":2},{"begin":415,"length":3},{"begin":420,"length":1},{"begin":422,"length":2},{"begin":425,"length":4},{"begin":432,"length":1},{"begin":438,"length":4},{"begin":443,"length":2},{"begin":446,"length":1},{"begin":450,"length":1},{"begin":454,"length":1},{"begin":456,"length":2},{"begin":460,"length":2},{"begin":463,"length":3},{"begin":467,"length":1},{"begin":469,"length":2},{"begin":473,"length":1},{"begin":478,"length":1},{"begin":480,"length":1},{"begin":482,"length":8},{"begin":492,"length":4},{"begin":497,"length":1},{"begin":499,"length":2},{"begin":502,"length":2},{"begin":506,"length":1},{"begin":509,"length":1},{"begin":512,"length":3},{"begin":517,"length":1},{"begin":520,"length":3},{"begin":529,"length":4},{"begin":534,"length":1},{"begin":537,"length":1},{"begin":539,"length":3},{"begin":543,"length":8},{"begin":552,"length":1},{"begin":555,"length":2},{"begin":558,"length":2},{"begin":562,"length":2},{"begin":565,"length":2},{"begin":568,"length":1},{"begin":572,"length":3},{"begin":576,"length":4},{"begin":581,"length":1},{"begin":583,"length":1},{"begin":587,"length":1},{"begin":594,"length":1},{"begin":598,"length":1},{"begin":600,"length":5},{"begin":606,"length":1},{"begin":608,"length":1},{"begin":611,"length":1},{"begin":619,"length":2},{"begin":623,"length":3},{"begin":629,"length":2},{"begin":632,"length":1},{"begin":634,"length":1},{"begin":636,"length":2},{"begin":640,"length":2},{"begin":644,"length":1},{"begin":648,"length":3},{"begin":652,"length":1},{"begin":657,"length":1},{"begin":659,"length":1},{"begin":671,"length":1},{"begin":673,"length":3},{"begin":678,"length":1},{"begin":685,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-06-08T07:31:40.987 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~39s , 2023-06-08T07:31:41.135 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json Ignoring malparsed backtrace: 2023-06-08T07:32:38.457 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":724,"fsid":"87dea55e-f8c6-48e7-a5de-497a76d690ca","created":"2023-06-08T07:19:03.434687+0000","modified":"2023-06-08T07:31:37.763740+0000","last_up_change":"2023-06-08T07:31:23.586612+0000","last_in_change":"2023-06-08T07:19:05.746653+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-06-08T07:19:11.856148+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-06-08T07:19:14.054099+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-06-08T07:19:21.041786+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"716","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":693,"snap_epoch":716,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"bc37cf1c-3e05-46a6-8dbc-b3e1a597b390","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":16,"down_at":711,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6800","nonce":2219628572}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6801","nonce":2219628572}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2219628572}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6805","nonce":2219628572}]},"public_addr":"172.21.15.99:6800/2219628572","cluster_addr":"172.21.15.99:6801/2219628572","heartbeat_back_addr":"172.21.15.99:6804/2219628572","heartbeat_front_addr":"172.21.15.99:6805/2219628572","state":["exists"]},{"osd":1,"uuid":"2628ebee-943b-4c37-b142-f4c7ab979a56","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":786871654}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6803","nonce":786871654}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6807","nonce":786871654}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":786871654}]},"public_addr":"172.21.15.99:6802/786871654","cluster_addr":"172.21.15.99:6803/786871654","heartbeat_back_addr":"172.21.15.99:6807/786871654","heartbeat_front_addr":"172.21.15.99:6806/786871654","state":["exists","up"]},{"osd":2,"uuid":"de6b6a32-e5a8-478f-8b75-94aebfb2b717","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6803","nonce":3055208471}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6802","nonce":3055208471}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6807","nonce":3055208471}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6808","nonce":3055208471}]},"public_addr":"172.21.15.162:6803/3055208471","cluster_addr":"172.21.15.162:6802/3055208471","heartbeat_back_addr":"172.21.15.162:6807/3055208471","heartbeat_front_addr":"172.21.15.162:6808/3055208471","state":["exists","up"]},{"osd":3,"uuid":"02b56dc3-6c99-49d7-b3d6-d5253d66b5c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6805","nonce":2372637129}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6804","nonce":2372637129}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6806","nonce":2372637129}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6809","nonce":2372637129}]},"public_addr":"172.21.15.162:6805/2372637129","cluster_addr":"172.21.15.162:6804/2372637129","heartbeat_back_addr":"172.21.15.162:6806/2372637129","heartbeat_front_addr":"172.21.15.162:6809/2372637129","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-06-08T07:31:23.586612+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":1},{"begin":8,"length":3},{"begin":14,"length":1},{"begin":16,"length":2},{"begin":20,"length":1},{"begin":22,"length":4},{"begin":27,"length":1},{"begin":31,"length":2},{"begin":36,"length":3},{"begin":40,"length":1},{"begin":42,"length":2},{"begin":45,"length":1},{"begin":49,"length":1},{"begin":51,"length":2},{"begin":55,"length":1},{"begin":57,"length":3},{"begin":61,"length":2},{"begin":64,"length":1},{"begin":68,"length":4},{"begin":75,"length":1},{"begin":77,"length":4},{"begin":82,"length":2},{"begin":86,"length":1},{"begin":89,"length":1},{"begin":91,"length":2},{"begin":94,"length":6},{"begin":101,"length":3},{"begin":106,"length":1},{"begin":108,"length":1},{"begin":111,"length":2},{"begin":114,"length":3},{"begin":118,"length":2},{"begin":121,"length":2},{"begin":124,"length":1},{"begin":135,"length":2},{"begin":138,"length":2},{"begin":142,"length":1},{"begin":145,"length":2},{"begin":148,"length":1},{"begin":150,"length":3},{"begin":160,"length":1},{"begin":162,"length":1},{"begin":165,"length":1},{"begin":169,"length":1},{"begin":171,"length":1},{"begin":173,"length":1},{"begin":176,"length":1},{"begin":178,"length":3},{"begin":184,"length":2},{"begin":188,"length":1},{"begin":190,"length":1},{"begin":192,"length":1},{"begin":194,"length":2},{"begin":198,"length":2},{"begin":202,"length":3},{"begin":208,"length":4},{"begin":215,"length":1},{"begin":218,"length":4},{"begin":226,"length":1},{"begin":228,"length":3},{"begin":233,"length":5},{"begin":240,"length":1},{"begin":243,"length":1},{"begin":246,"length":1},{"begin":248,"length":2},{"begin":251,"length":1},{"begin":253,"length":1},{"begin":256,"length":3},{"begin":263,"length":1},{"begin":266,"length":3},{"begin":270,"length":2},{"begin":274,"length":2},{"begin":277,"length":1},{"begin":280,"length":1},{"begin":284,"length":3},{"begin":288,"length":2},{"begin":291,"length":2},{"begin":295,"length":1},{"begin":297,"length":2},{"begin":303,"length":3},{"begin":307,"length":1},{"begin":309,"length":3},{"begin":313,"length":1},{"begin":316,"length":1},{"begin":319,"length":2},{"begin":323,"length":1},{"begin":326,"length":2},{"begin":329,"length":7},{"begin":338,"length":1},{"begin":342,"length":1},{"begin":345,"length":2},{"begin":348,"length":3},{"begin":353,"length":1},{"begin":357,"length":4},{"begin":363,"length":2},{"begin":369,"length":1},{"begin":373,"length":3},{"begin":377,"length":2},{"begin":380,"length":1},{"begin":382,"length":1},{"begin":385,"length":4},{"begin":393,"length":1},{"begin":396,"length":1},{"begin":401,"length":1},{"begin":403,"length":1},{"begin":405,"length":1},{"begin":407,"length":1},{"begin":409,"length":1},{"begin":411,"length":2},{"begin":415,"length":3},{"begin":420,"length":1},{"begin":422,"length":2},{"begin":425,"length":4},{"begin":432,"length":1},{"begin":438,"length":4},{"begin":443,"length":2},{"begin":446,"length":1},{"begin":450,"length":1},{"begin":454,"length":1},{"begin":456,"length":2},{"begin":460,"length":2},{"begin":463,"length":3},{"begin":467,"length":1},{"begin":469,"length":2},{"begin":473,"length":1},{"begin":478,"length":1},{"begin":480,"length":1},{"begin":482,"length":8},{"begin":492,"length":4},{"begin":497,"length":1},{"begin":499,"length":2},{"begin":502,"length":2},{"begin":506,"length":1},{"begin":509,"length":1},{"begin":512,"length":3},{"begin":517,"length":1},{"begin":520,"length":3},{"begin":529,"length":4},{"begin":534,"length":1},{"begin":537,"length":1},{"begin":539,"length":3},{"begin":543,"length":8},{"begin":552,"length":1},{"begin":555,"length":2},{"begin":558,"length":2},{"begin":562,"length":2},{"begin":565,"length":2},{"begin":568,"length":1},{"begin":572,"length":3},{"begin":576,"length":4},{"begin":581,"length":1},{"begin":583,"length":1},{"begin":587,"length":1},{"begin":594,"length":1},{"begin":598,"length":1},{"begin":600,"length":5},{"begin":606,"length":1},{"begin":608,"length":1},{"begin":611,"length":1},{"begin":619,"length":2},{"begin":623,"length":3},{"begin":629,"length":2},{"begin":632,"length":1},{"begin":634,"length":1},{"begin":636,"length":2},{"begin":640,"length":2},{"begin":644,"length":1},{"begin":648,"length":3},{"begin":652,"length":1},{"begin":657,"length":1},{"begin":659,"length":1},{"begin":671,"length":1},{"begin":673,"length":3},{"begin":678,"length":1},{"begin":685,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-06-08T07:32:41.473 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-06-08T07:32:41.544 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~99s Ignoring malparsed backtrace: 2023-06-08T07:33:39.028 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":724,"fsid":"87dea55e-f8c6-48e7-a5de-497a76d690ca","created":"2023-06-08T07:19:03.434687+0000","modified":"2023-06-08T07:31:37.763740+0000","last_up_change":"2023-06-08T07:31:23.586612+0000","last_in_change":"2023-06-08T07:19:05.746653+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-06-08T07:19:11.856148+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-06-08T07:19:14.054099+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-06-08T07:19:21.041786+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"716","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":693,"snap_epoch":716,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"bc37cf1c-3e05-46a6-8dbc-b3e1a597b390","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":16,"down_at":711,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6800","nonce":2219628572}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6801","nonce":2219628572}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2219628572}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6805","nonce":2219628572}]},"public_addr":"172.21.15.99:6800/2219628572","cluster_addr":"172.21.15.99:6801/2219628572","heartbeat_back_addr":"172.21.15.99:6804/2219628572","heartbeat_front_addr":"172.21.15.99:6805/2219628572","state":["exists"]},{"osd":1,"uuid":"2628ebee-943b-4c37-b142-f4c7ab979a56","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":786871654}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6803","nonce":786871654}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6807","nonce":786871654}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":786871654}]},"public_addr":"172.21.15.99:6802/786871654","cluster_addr":"172.21.15.99:6803/786871654","heartbeat_back_addr":"172.21.15.99:6807/786871654","heartbeat_front_addr":"172.21.15.99:6806/786871654","state":["exists","up"]},{"osd":2,"uuid":"de6b6a32-e5a8-478f-8b75-94aebfb2b717","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6803","nonce":3055208471}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6802","nonce":3055208471}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6807","nonce":3055208471}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6808","nonce":3055208471}]},"public_addr":"172.21.15.162:6803/3055208471","cluster_addr":"172.21.15.162:6802/3055208471","heartbeat_back_addr":"172.21.15.162:6807/3055208471","heartbeat_front_addr":"172.21.15.162:6808/3055208471","state":["exists","up"]},{"osd":3,"uuid":"02b56dc3-6c99-49d7-b3d6-d5253d66b5c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6805","nonce":2372637129}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6804","nonce":2372637129}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6806","nonce":2372637129}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6809","nonce":2372637129}]},"public_addr":"172.21.15.162:6805/2372637129","cluster_addr":"172.21.15.162:6804/2372637129","heartbeat_back_addr":"172.21.15.162:6806/2372637129","heartbeat_front_addr":"172.21.15.162:6809/2372637129","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-06-08T07:31:23.586612+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":1},{"begin":8,"length":3},{"begin":14,"length":1},{"begin":16,"length":2},{"begin":20,"length":1},{"begin":22,"length":4},{"begin":27,"length":1},{"begin":31,"length":2},{"begin":36,"length":3},{"begin":40,"length":1},{"begin":42,"length":2},{"begin":45,"length":1},{"begin":49,"length":1},{"begin":51,"length":2},{"begin":55,"length":1},{"begin":57,"length":3},{"begin":61,"length":2},{"begin":64,"length":1},{"begin":68,"length":4},{"begin":75,"length":1},{"begin":77,"length":4},{"begin":82,"length":2},{"begin":86,"length":1},{"begin":89,"length":1},{"begin":91,"length":2},{"begin":94,"length":6},{"begin":101,"length":3},{"begin":106,"length":1},{"begin":108,"length":1},{"begin":111,"length":2},{"begin":114,"length":3},{"begin":118,"length":2},{"begin":121,"length":2},{"begin":124,"length":1},{"begin":135,"length":2},{"begin":138,"length":2},{"begin":142,"length":1},{"begin":145,"length":2},{"begin":148,"length":1},{"begin":150,"length":3},{"begin":160,"length":1},{"begin":162,"length":1},{"begin":165,"length":1},{"begin":169,"length":1},{"begin":171,"length":1},{"begin":173,"length":1},{"begin":176,"length":1},{"begin":178,"length":3},{"begin":184,"length":2},{"begin":188,"length":1},{"begin":190,"length":1},{"begin":192,"length":1},{"begin":194,"length":2},{"begin":198,"length":2},{"begin":202,"length":3},{"begin":208,"length":4},{"begin":215,"length":1},{"begin":218,"length":4},{"begin":226,"length":1},{"begin":228,"length":3},{"begin":233,"length":5},{"begin":240,"length":1},{"begin":243,"length":1},{"begin":246,"length":1},{"begin":248,"length":2},{"begin":251,"length":1},{"begin":253,"length":1},{"begin":256,"length":3},{"begin":263,"length":1},{"begin":266,"length":3},{"begin":270,"length":2},{"begin":274,"length":2},{"begin":277,"length":1},{"begin":280,"length":1},{"begin":284,"length":3},{"begin":288,"length":2},{"begin":291,"length":2},{"begin":295,"length":1},{"begin":297,"length":2},{"begin":303,"length":3},{"begin":307,"length":1},{"begin":309,"length":3},{"begin":313,"length":1},{"begin":316,"length":1},{"begin":319,"length":2},{"begin":323,"length":1},{"begin":326,"length":2},{"begin":329,"length":7},{"begin":338,"length":1},{"begin":342,"length":1},{"begin":345,"length":2},{"begin":348,"length":3},{"begin":353,"length":1},{"begin":357,"length":4},{"begin":363,"length":2},{"begin":369,"length":1},{"begin":373,"length":3},{"begin":377,"length":2},{"begin":380,"length":1},{"begin":382,"length":1},{"begin":385,"length":4},{"begin":393,"length":1},{"begin":396,"length":1},{"begin":401,"length":1},{"begin":403,"length":1},{"begin":405,"length":1},{"begin":407,"length":1},{"begin":409,"length":1},{"begin":411,"length":2},{"begin":415,"length":3},{"begin":420,"length":1},{"begin":422,"length":2},{"begin":425,"length":4},{"begin":432,"length":1},{"begin":438,"length":4},{"begin":443,"length":2},{"begin":446,"length":1},{"begin":450,"length":1},{"begin":454,"length":1},{"begin":456,"length":2},{"begin":460,"length":2},{"begin":463,"length":3},{"begin":467,"length":1},{"begin":469,"length":2},{"begin":473,"length":1},{"begin":478,"length":1},{"begin":480,"length":1},{"begin":482,"length":8},{"begin":492,"length":4},{"begin":497,"length":1},{"begin":499,"length":2},{"begin":502,"length":2},{"begin":506,"length":1},{"begin":509,"length":1},{"begin":512,"length":3},{"begin":517,"length":1},{"begin":520,"length":3},{"begin":529,"length":4},{"begin":534,"length":1},{"begin":537,"length":1},{"begin":539,"length":3},{"begin":543,"length":8},{"begin":552,"length":1},{"begin":555,"length":2},{"begin":558,"length":2},{"begin":562,"length":2},{"begin":565,"length":2},{"begin":568,"length":1},{"begin":572,"length":3},{"begin":576,"length":4},{"begin":581,"length":1},{"begin":583,"length":1},{"begin":587,"length":1},{"begin":594,"length":1},{"begin":598,"length":1},{"begin":600,"length":5},{"begin":606,"length":1},{"begin":608,"length":1},{"begin":611,"length":1},{"begin":619,"length":2},{"begin":623,"length":3},{"begin":629,"length":2},{"begin":632,"length":1},{"begin":634,"length":1},{"begin":636,"length":2},{"begin":640,"length":2},{"begin":644,"length":1},{"begin":648,"length":3},{"begin":652,"length":1},{"begin":657,"length":1},{"begin":659,"length":1},{"begin":671,"length":1},{"begin":673,"length":3},{"begin":678,"length":1},{"begin":685,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-06-08T07:33:42.044 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-06-08T07:33:42.091 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~160s Ignoring malparsed backtrace: 2023-06-08T07:34:39.671 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":724,"fsid":"87dea55e-f8c6-48e7-a5de-497a76d690ca","created":"2023-06-08T07:19:03.434687+0000","modified":"2023-06-08T07:31:37.763740+0000","last_up_change":"2023-06-08T07:31:23.586612+0000","last_in_change":"2023-06-08T07:19:05.746653+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-06-08T07:19:11.856148+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-06-08T07:19:14.054099+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-06-08T07:19:21.041786+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"716","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":693,"snap_epoch":716,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"bc37cf1c-3e05-46a6-8dbc-b3e1a597b390","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":16,"down_at":711,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6800","nonce":2219628572}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6801","nonce":2219628572}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2219628572}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6805","nonce":2219628572}]},"public_addr":"172.21.15.99:6800/2219628572","cluster_addr":"172.21.15.99:6801/2219628572","heartbeat_back_addr":"172.21.15.99:6804/2219628572","heartbeat_front_addr":"172.21.15.99:6805/2219628572","state":["exists"]},{"osd":1,"uuid":"2628ebee-943b-4c37-b142-f4c7ab979a56","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":786871654}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6803","nonce":786871654}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6807","nonce":786871654}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":786871654}]},"public_addr":"172.21.15.99:6802/786871654","cluster_addr":"172.21.15.99:6803/786871654","heartbeat_back_addr":"172.21.15.99:6807/786871654","heartbeat_front_addr":"172.21.15.99:6806/786871654","state":["exists","up"]},{"osd":2,"uuid":"de6b6a32-e5a8-478f-8b75-94aebfb2b717","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6803","nonce":3055208471}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6802","nonce":3055208471}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6807","nonce":3055208471}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6808","nonce":3055208471}]},"public_addr":"172.21.15.162:6803/3055208471","cluster_addr":"172.21.15.162:6802/3055208471","heartbeat_back_addr":"172.21.15.162:6807/3055208471","heartbeat_front_addr":"172.21.15.162:6808/3055208471","state":["exists","up"]},{"osd":3,"uuid":"02b56dc3-6c99-49d7-b3d6-d5253d66b5c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6805","nonce":2372637129}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6804","nonce":2372637129}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6806","nonce":2372637129}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6809","nonce":2372637129}]},"public_addr":"172.21.15.162:6805/2372637129","cluster_addr":"172.21.15.162:6804/2372637129","heartbeat_back_addr":"172.21.15.162:6806/2372637129","heartbeat_front_addr":"172.21.15.162:6809/2372637129","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-06-08T07:31:23.586612+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":1},{"begin":8,"length":3},{"begin":14,"length":1},{"begin":16,"length":2},{"begin":20,"length":1},{"begin":22,"length":4},{"begin":27,"length":1},{"begin":31,"length":2},{"begin":36,"length":3},{"begin":40,"length":1},{"begin":42,"length":2},{"begin":45,"length":1},{"begin":49,"length":1},{"begin":51,"length":2},{"begin":55,"length":1},{"begin":57,"length":3},{"begin":61,"length":2},{"begin":64,"length":1},{"begin":68,"length":4},{"begin":75,"length":1},{"begin":77,"length":4},{"begin":82,"length":2},{"begin":86,"length":1},{"begin":89,"length":1},{"begin":91,"length":2},{"begin":94,"length":6},{"begin":101,"length":3},{"begin":106,"length":1},{"begin":108,"length":1},{"begin":111,"length":2},{"begin":114,"length":3},{"begin":118,"length":2},{"begin":121,"length":2},{"begin":124,"length":1},{"begin":135,"length":2},{"begin":138,"length":2},{"begin":142,"length":1},{"begin":145,"length":2},{"begin":148,"length":1},{"begin":150,"length":3},{"begin":160,"length":1},{"begin":162,"length":1},{"begin":165,"length":1},{"begin":169,"length":1},{"begin":171,"length":1},{"begin":173,"length":1},{"begin":176,"length":1},{"begin":178,"length":3},{"begin":184,"length":2},{"begin":188,"length":1},{"begin":190,"length":1},{"begin":192,"length":1},{"begin":194,"length":2},{"begin":198,"length":2},{"begin":202,"length":3},{"begin":208,"length":4},{"begin":215,"length":1},{"begin":218,"length":4},{"begin":226,"length":1},{"begin":228,"length":3},{"begin":233,"length":5},{"begin":240,"length":1},{"begin":243,"length":1},{"begin":246,"length":1},{"begin":248,"length":2},{"begin":251,"length":1},{"begin":253,"length":1},{"begin":256,"length":3},{"begin":263,"length":1},{"begin":266,"length":3},{"begin":270,"length":2},{"begin":274,"length":2},{"begin":277,"length":1},{"begin":280,"length":1},{"begin":284,"length":3},{"begin":288,"length":2},{"begin":291,"length":2},{"begin":295,"length":1},{"begin":297,"length":2},{"begin":303,"length":3},{"begin":307,"length":1},{"begin":309,"length":3},{"begin":313,"length":1},{"begin":316,"length":1},{"begin":319,"length":2},{"begin":323,"length":1},{"begin":326,"length":2},{"begin":329,"length":7},{"begin":338,"length":1},{"begin":342,"length":1},{"begin":345,"length":2},{"begin":348,"length":3},{"begin":353,"length":1},{"begin":357,"length":4},{"begin":363,"length":2},{"begin":369,"length":1},{"begin":373,"length":3},{"begin":377,"length":2},{"begin":380,"length":1},{"begin":382,"length":1},{"begin":385,"length":4},{"begin":393,"length":1},{"begin":396,"length":1},{"begin":401,"length":1},{"begin":403,"length":1},{"begin":405,"length":1},{"begin":407,"length":1},{"begin":409,"length":1},{"begin":411,"length":2},{"begin":415,"length":3},{"begin":420,"length":1},{"begin":422,"length":2},{"begin":425,"length":4},{"begin":432,"length":1},{"begin":438,"length":4},{"begin":443,"length":2},{"begin":446,"length":1},{"begin":450,"length":1},{"begin":454,"length":1},{"begin":456,"length":2},{"begin":460,"length":2},{"begin":463,"length":3},{"begin":467,"length":1},{"begin":469,"length":2},{"begin":473,"length":1},{"begin":478,"length":1},{"begin":480,"length":1},{"begin":482,"length":8},{"begin":492,"length":4},{"begin":497,"length":1},{"begin":499,"length":2},{"begin":502,"length":2},{"begin":506,"length":1},{"begin":509,"length":1},{"begin":512,"length":3},{"begin":517,"length":1},{"begin":520,"length":3},{"begin":529,"length":4},{"begin":534,"length":1},{"begin":537,"length":1},{"begin":539,"length":3},{"begin":543,"length":8},{"begin":552,"length":1},{"begin":555,"length":2},{"begin":558,"length":2},{"begin":562,"length":2},{"begin":565,"length":2},{"begin":568,"length":1},{"begin":572,"length":3},{"begin":576,"length":4},{"begin":581,"length":1},{"begin":583,"length":1},{"begin":587,"length":1},{"begin":594,"length":1},{"begin":598,"length":1},{"begin":600,"length":5},{"begin":606,"length":1},{"begin":608,"length":1},{"begin":611,"length":1},{"begin":619,"length":2},{"begin":623,"length":3},{"begin":629,"length":2},{"begin":632,"length":1},{"begin":634,"length":1},{"begin":636,"length":2},{"begin":640,"length":2},{"begin":644,"length":1},{"begin":648,"length":3},{"begin":652,"length":1},{"begin":657,"length":1},{"begin":659,"length":1},{"begin":671,"length":1},{"begin":673,"length":3},{"begin":678,"length":1},{"begin":685,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-06-08T07:34:42.638 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~220s , 2023-06-08T07:34:42.687 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json Ignoring malparsed backtrace: 2023-06-08T07:35:40.314 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":724,"fsid":"87dea55e-f8c6-48e7-a5de-497a76d690ca","created":"2023-06-08T07:19:03.434687+0000","modified":"2023-06-08T07:31:37.763740+0000","last_up_change":"2023-06-08T07:31:23.586612+0000","last_in_change":"2023-06-08T07:19:05.746653+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-06-08T07:19:11.856148+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-06-08T07:19:14.054099+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-06-08T07:19:21.041786+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"716","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":693,"snap_epoch":716,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"bc37cf1c-3e05-46a6-8dbc-b3e1a597b390","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":16,"down_at":711,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6800","nonce":2219628572}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6801","nonce":2219628572}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2219628572}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6805","nonce":2219628572}]},"public_addr":"172.21.15.99:6800/2219628572","cluster_addr":"172.21.15.99:6801/2219628572","heartbeat_back_addr":"172.21.15.99:6804/2219628572","heartbeat_front_addr":"172.21.15.99:6805/2219628572","state":["exists"]},{"osd":1,"uuid":"2628ebee-943b-4c37-b142-f4c7ab979a56","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":786871654}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6803","nonce":786871654}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6807","nonce":786871654}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":786871654}]},"public_addr":"172.21.15.99:6802/786871654","cluster_addr":"172.21.15.99:6803/786871654","heartbeat_back_addr":"172.21.15.99:6807/786871654","heartbeat_front_addr":"172.21.15.99:6806/786871654","state":["exists","up"]},{"osd":2,"uuid":"de6b6a32-e5a8-478f-8b75-94aebfb2b717","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6803","nonce":3055208471}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6802","nonce":3055208471}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6807","nonce":3055208471}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6808","nonce":3055208471}]},"public_addr":"172.21.15.162:6803/3055208471","cluster_addr":"172.21.15.162:6802/3055208471","heartbeat_back_addr":"172.21.15.162:6807/3055208471","heartbeat_front_addr":"172.21.15.162:6808/3055208471","state":["exists","up"]},{"osd":3,"uuid":"02b56dc3-6c99-49d7-b3d6-d5253d66b5c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6805","nonce":2372637129}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6804","nonce":2372637129}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6806","nonce":2372637129}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6809","nonce":2372637129}]},"public_addr":"172.21.15.162:6805/2372637129","cluster_addr":"172.21.15.162:6804/2372637129","heartbeat_back_addr":"172.21.15.162:6806/2372637129","heartbeat_front_addr":"172.21.15.162:6809/2372637129","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-06-08T07:31:23.586612+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":1},{"begin":8,"length":3},{"begin":14,"length":1},{"begin":16,"length":2},{"begin":20,"length":1},{"begin":22,"length":4},{"begin":27,"length":1},{"begin":31,"length":2},{"begin":36,"length":3},{"begin":40,"length":1},{"begin":42,"length":2},{"begin":45,"length":1},{"begin":49,"length":1},{"begin":51,"length":2},{"begin":55,"length":1},{"begin":57,"length":3},{"begin":61,"length":2},{"begin":64,"length":1},{"begin":68,"length":4},{"begin":75,"length":1},{"begin":77,"length":4},{"begin":82,"length":2},{"begin":86,"length":1},{"begin":89,"length":1},{"begin":91,"length":2},{"begin":94,"length":6},{"begin":101,"length":3},{"begin":106,"length":1},{"begin":108,"length":1},{"begin":111,"length":2},{"begin":114,"length":3},{"begin":118,"length":2},{"begin":121,"length":2},{"begin":124,"length":1},{"begin":135,"length":2},{"begin":138,"length":2},{"begin":142,"length":1},{"begin":145,"length":2},{"begin":148,"length":1},{"begin":150,"length":3},{"begin":160,"length":1},{"begin":162,"length":1},{"begin":165,"length":1},{"begin":169,"length":1},{"begin":171,"length":1},{"begin":173,"length":1},{"begin":176,"length":1},{"begin":178,"length":3},{"begin":184,"length":2},{"begin":188,"length":1},{"begin":190,"length":1},{"begin":192,"length":1},{"begin":194,"length":2},{"begin":198,"length":2},{"begin":202,"length":3},{"begin":208,"length":4},{"begin":215,"length":1},{"begin":218,"length":4},{"begin":226,"length":1},{"begin":228,"length":3},{"begin":233,"length":5},{"begin":240,"length":1},{"begin":243,"length":1},{"begin":246,"length":1},{"begin":248,"length":2},{"begin":251,"length":1},{"begin":253,"length":1},{"begin":256,"length":3},{"begin":263,"length":1},{"begin":266,"length":3},{"begin":270,"length":2},{"begin":274,"length":2},{"begin":277,"length":1},{"begin":280,"length":1},{"begin":284,"length":3},{"begin":288,"length":2},{"begin":291,"length":2},{"begin":295,"length":1},{"begin":297,"length":2},{"begin":303,"length":3},{"begin":307,"length":1},{"begin":309,"length":3},{"begin":313,"length":1},{"begin":316,"length":1},{"begin":319,"length":2},{"begin":323,"length":1},{"begin":326,"length":2},{"begin":329,"length":7},{"begin":338,"length":1},{"begin":342,"length":1},{"begin":345,"length":2},{"begin":348,"length":3},{"begin":353,"length":1},{"begin":357,"length":4},{"begin":363,"length":2},{"begin":369,"length":1},{"begin":373,"length":3},{"begin":377,"length":2},{"begin":380,"length":1},{"begin":382,"length":1},{"begin":385,"length":4},{"begin":393,"length":1},{"begin":396,"length":1},{"begin":401,"length":1},{"begin":403,"length":1},{"begin":405,"length":1},{"begin":407,"length":1},{"begin":409,"length":1},{"begin":411,"length":2},{"begin":415,"length":3},{"begin":420,"length":1},{"begin":422,"length":2},{"begin":425,"length":4},{"begin":432,"length":1},{"begin":438,"length":4},{"begin":443,"length":2},{"begin":446,"length":1},{"begin":450,"length":1},{"begin":454,"length":1},{"begin":456,"length":2},{"begin":460,"length":2},{"begin":463,"length":3},{"begin":467,"length":1},{"begin":469,"length":2},{"begin":473,"length":1},{"begin":478,"length":1},{"begin":480,"length":1},{"begin":482,"length":8},{"begin":492,"length":4},{"begin":497,"length":1},{"begin":499,"length":2},{"begin":502,"length":2},{"begin":506,"length":1},{"begin":509,"length":1},{"begin":512,"length":3},{"begin":517,"length":1},{"begin":520,"length":3},{"begin":529,"length":4},{"begin":534,"length":1},{"begin":537,"length":1},{"begin":539,"length":3},{"begin":543,"length":8},{"begin":552,"length":1},{"begin":555,"length":2},{"begin":558,"length":2},{"begin":562,"length":2},{"begin":565,"length":2},{"begin":568,"length":1},{"begin":572,"length":3},{"begin":576,"length":4},{"begin":581,"length":1},{"begin":583,"length":1},{"begin":587,"length":1},{"begin":594,"length":1},{"begin":598,"length":1},{"begin":600,"length":5},{"begin":606,"length":1},{"begin":608,"length":1},{"begin":611,"length":1},{"begin":619,"length":2},{"begin":623,"length":3},{"begin":629,"length":2},{"begin":632,"length":1},{"begin":634,"length":1},{"begin":636,"length":2},{"begin":640,"length":2},{"begin":644,"length":1},{"begin":648,"length":3},{"begin":652,"length":1},{"begin":657,"length":1},{"begin":659,"length":1},{"begin":671,"length":1},{"begin":673,"length":3},{"begin":678,"length":1},{"begin":685,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-06-08T07:35:43.185 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~281s , 2023-06-08T07:35:43.329 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json Ignoring malparsed backtrace: r = self._runner(client=self.ssh, name=self.shortname, **kwargs) , File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/orchestra/run.py", line 455, in run , r.wait() Ignoring malparsed backtrace: File "/home/teuthworker/src/github.com_ceph_ceph_ea807834105141b6938ce6a9dba28667420efa2e/qa/tasks/ceph.py", line 1469, in run_daemon , yield , File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/contextutil.py", line 32, in nested Ignoring malparsed backtrace: 2023-06-08T08:28:13.377 DEBUG:teuthology.orchestra.run.smithi099:> sync && sudo umount -f /var/lib/ceph/osd/ceph-1 , 2023-06-08T08:28:13.453 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-2 on ubuntu@smithi162.front.sepia.ceph.com , 2023-06-08T08:28:13.453 DEBUG:teuthology.orchestra.run.smithi162:> sync && sudo umount -f /var/lib/ceph/osd/ceph-2 Ignoring malparsed backtrace: /var/log/ceph/ceph-client.admin.127918.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.121150.log , 50.0% -- replaced with /var/log/ceph/ceph-client.admin.127918.log.gz , /var/log/ceph/ceph-client.admin.130308.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.123182.log Ignoring malparsed backtrace: 49.6% -- replaced with /var/log/ceph/ceph-client.admin.104245.log.gz , /var/log/ceph/ceph-client.admin.120435.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.117780.log , 50.0% -- replaced with /var/log/ceph/ceph-client.admin.120435.log.gz Ignoring malparsed backtrace: /var/log/ceph/ceph-client.admin.109652.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.124567.log , 50.0% -- replaced with /var/log/ceph/ceph-client.admin.109652.log.gz , /var/log/ceph/ceph-client.admin.104343.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.126564.log Ignoring malparsed backtrace: 50.0% -- replaced with /var/log/ceph/ceph-client.admin.128995.log.gz , /var/log/ceph/ceph-client.admin.122117.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.112523.log , 50.4% -- replaced with /var/log/ceph/ceph-client.admin.122117.log.gz Ignoring malparsed backtrace: /var/log/ceph/ceph-client.admin.125535.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.127679.log , 50.0% -- replaced with /var/log/ceph/ceph-client.admin.125535.log.gz , /var/log/ceph/ceph-client.admin.132894.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.103886.log Ignoring malparsed backtrace: /var/log/ceph/ceph-client.admin.111570.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.132032.log , 50.4% -- replaced with /var/log/ceph/ceph-client.admin.111570.log.gz , /var/log/ceph/ceph-client.admin.105643.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.135368.log Ignoring malparsed backtrace: 49.6% -- replaced with /var/log/ceph/ceph-client.admin.115634.log.gz , /var/log/ceph/ceph-client.admin.105211.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.110324.log , 50.0% -- replaced with /var/log/ceph/ceph-client.admin.105211.log.gz Ignoring malparsed backtrace: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.114661.log , /var/log/ceph/ceph-client.admin.103574.log: 50.0% -- replaced with /var/log/ceph/ceph-client.admin.103574.log.gz , /var/log/ceph/ceph-client.admin.133490.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.110814.log Ignoring malparsed backtrace: 50.4% -- replaced with /var/log/ceph/ceph-client.admin.112250.log.gz , /var/log/ceph/ceph-client.admin.120716.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.118263.log , 50.0% -- replaced with /var/log/ceph/ceph-client.admin.120716.log.gz Ignoring malparsed backtrace: 50.0% -- replaced with /var/log/ceph/ceph-client.admin.131038.log.gz , /var/log/ceph/ceph-client.admin.126772.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.111997.log , 50.0% -- replaced with /var/log/ceph/ceph-client.admin.126772.log.gz Ignoring malparsed backtrace: /var/log/ceph/ceph-client.admin.111786.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.132208.log , 49.2% -- replaced with /var/log/ceph/ceph-client.admin.111786.log.gz , /var/log/ceph/ceph-client.admin.113152.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.109491.log Ignoring malparsed backtrace: 50.0% -- replaced with /var/log/ceph/ceph-client.admin.114283.log.gz , /var/log/ceph/ceph-client.admin.125913.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.105037.log , 50.4% -- replaced with /var/log/ceph/ceph-client.admin.125913.log.gz Ignoring malparsed backtrace: sys 0m0.444s , 92.7% -- replaced with /var/log/ceph/ceph-osd.3.log.gz , Ignoring malparsed backtrace: 2023-06-08T08:28:21.213 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive , 2023-06-08T08:28:21.227 INFO:teuthology.task.internal:Transferring archived files... , 2023-06-08T08:28:21.229 DEBUG:teuthology.misc:Transferring archived files from smithi099:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/matan-2023-06-08_06:36:16-crimson-rados-wip-matanb-crimson-testing-bug-61504-distro-crimson-smithi/7298180/remote/smithi099 Ignoring malparsed backtrace: ceph version 18.0.0-4330-g9b3f3761 (9b3f3761ef331c3cedab822629c5de78b7d54292) reef (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7f07f68f8ed4] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x1b0) [0x5570926960ca] Ignoring malparsed backtrace: 2023-06-08T07:31:38.120 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":724,"fsid":"87dea55e-f8c6-48e7-a5de-497a76d690ca","created":"2023-06-08T07:19:03.434687+0000","modified":"2023-06-08T07:31:37.763740+0000","last_up_change":"2023-06-08T07:31:23.586612+0000","last_in_change":"2023-06-08T07:19:05.746653+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-06-08T07:19:11.856148+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-06-08T07:19:14.054099+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-06-08T07:19:21.041786+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"716","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":693,"snap_epoch":716,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"bc37cf1c-3e05-46a6-8dbc-b3e1a597b390","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":16,"down_at":711,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6800","nonce":2219628572}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6801","nonce":2219628572}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2219628572}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6805","nonce":2219628572}]},"public_addr":"172.21.15.99:6800/2219628572","cluster_addr":"172.21.15.99:6801/2219628572","heartbeat_back_addr":"172.21.15.99:6804/2219628572","heartbeat_front_addr":"172.21.15.99:6805/2219628572","state":["exists"]},{"osd":1,"uuid":"2628ebee-943b-4c37-b142-f4c7ab979a56","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":786871654}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6803","nonce":786871654}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6807","nonce":786871654}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":786871654}]},"public_addr":"172.21.15.99:6802/786871654","cluster_addr":"172.21.15.99:6803/786871654","heartbeat_back_addr":"172.21.15.99:6807/786871654","heartbeat_front_addr":"172.21.15.99:6806/786871654","state":["exists","up"]},{"osd":2,"uuid":"de6b6a32-e5a8-478f-8b75-94aebfb2b717","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6803","nonce":3055208471}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6802","nonce":3055208471}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6807","nonce":3055208471}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6808","nonce":3055208471}]},"public_addr":"172.21.15.162:6803/3055208471","cluster_addr":"172.21.15.162:6802/3055208471","heartbeat_back_addr":"172.21.15.162:6807/3055208471","heartbeat_front_addr":"172.21.15.162:6808/3055208471","state":["exists","up"]},{"osd":3,"uuid":"02b56dc3-6c99-49d7-b3d6-d5253d66b5c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6805","nonce":2372637129}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6804","nonce":2372637129}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6806","nonce":2372637129}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6809","nonce":2372637129}]},"public_addr":"172.21.15.162:6805/2372637129","cluster_addr":"172.21.15.162:6804/2372637129","heartbeat_back_addr":"172.21.15.162:6806/2372637129","heartbeat_front_addr":"172.21.15.162:6809/2372637129","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-06-08T07:31:23.586612+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":1},{"begin":8,"length":3},{"begin":14,"length":1},{"begin":16,"length":2},{"begin":20,"length":1},{"begin":22,"length":4},{"begin":27,"length":1},{"begin":31,"length":2},{"begin":36,"length":3},{"begin":40,"length":1},{"begin":42,"length":2},{"begin":45,"length":1},{"begin":49,"length":1},{"begin":51,"length":2},{"begin":55,"length":1},{"begin":57,"length":3},{"begin":61,"length":2},{"begin":64,"length":1},{"begin":68,"length":4},{"begin":75,"length":1},{"begin":77,"length":4},{"begin":82,"length":2},{"begin":86,"length":1},{"begin":89,"length":1},{"begin":91,"length":2},{"begin":94,"length":6},{"begin":101,"length":3},{"begin":106,"length":1},{"begin":108,"length":1},{"begin":111,"length":2},{"begin":114,"length":3},{"begin":118,"length":2},{"begin":121,"length":2},{"begin":124,"length":1},{"begin":135,"length":2},{"begin":138,"length":2},{"begin":142,"length":1},{"begin":145,"length":2},{"begin":148,"length":1},{"begin":150,"length":3},{"begin":160,"length":1},{"begin":162,"length":1},{"begin":165,"length":1},{"begin":169,"length":1},{"begin":171,"length":1},{"begin":173,"length":1},{"begin":176,"length":1},{"begin":178,"length":3},{"begin":184,"length":2},{"begin":188,"length":1},{"begin":190,"length":1},{"begin":192,"length":1},{"begin":194,"length":2},{"begin":198,"length":2},{"begin":202,"length":3},{"begin":208,"length":4},{"begin":215,"length":1},{"begin":218,"length":4},{"begin":226,"length":1},{"begin":228,"length":3},{"begin":233,"length":5},{"begin":240,"length":1},{"begin":243,"length":1},{"begin":246,"length":1},{"begin":248,"length":2},{"begin":251,"length":1},{"begin":253,"length":1},{"begin":256,"length":3},{"begin":263,"length":1},{"begin":266,"length":3},{"begin":270,"length":2},{"begin":274,"length":2},{"begin":277,"length":1},{"begin":280,"length":1},{"begin":284,"length":3},{"begin":288,"length":2},{"begin":291,"length":2},{"begin":295,"length":1},{"begin":297,"length":2},{"begin":303,"length":3},{"begin":307,"length":1},{"begin":309,"length":3},{"begin":313,"length":1},{"begin":316,"length":1},{"begin":319,"length":2},{"begin":323,"length":1},{"begin":326,"length":2},{"begin":329,"length":7},{"begin":338,"length":1},{"begin":342,"length":1},{"begin":345,"length":2},{"begin":348,"length":3},{"begin":353,"length":1},{"begin":357,"length":4},{"begin":363,"length":2},{"begin":369,"length":1},{"begin":373,"length":3},{"begin":377,"length":2},{"begin":380,"length":1},{"begin":382,"length":1},{"begin":385,"length":4},{"begin":393,"length":1},{"begin":396,"length":1},{"begin":401,"length":1},{"begin":403,"length":1},{"begin":405,"length":1},{"begin":407,"length":1},{"begin":409,"length":1},{"begin":411,"length":2},{"begin":415,"length":3},{"begin":420,"length":1},{"begin":422,"length":2},{"begin":425,"length":4},{"begin":432,"length":1},{"begin":438,"length":4},{"begin":443,"length":2},{"begin":446,"length":1},{"begin":450,"length":1},{"begin":454,"length":1},{"begin":456,"length":2},{"begin":460,"length":2},{"begin":463,"length":3},{"begin":467,"length":1},{"begin":469,"length":2},{"begin":473,"length":1},{"begin":478,"length":1},{"begin":480,"length":1},{"begin":482,"length":8},{"begin":492,"length":4},{"begin":497,"length":1},{"begin":499,"length":2},{"begin":502,"length":2},{"begin":506,"length":1},{"begin":509,"length":1},{"begin":512,"length":3},{"begin":517,"length":1},{"begin":520,"length":3},{"begin":529,"length":4},{"begin":534,"length":1},{"begin":537,"length":1},{"begin":539,"length":3},{"begin":543,"length":8},{"begin":552,"length":1},{"begin":555,"length":2},{"begin":558,"length":2},{"begin":562,"length":2},{"begin":565,"length":2},{"begin":568,"length":1},{"begin":572,"length":3},{"begin":576,"length":4},{"begin":581,"length":1},{"begin":583,"length":1},{"begin":587,"length":1},{"begin":594,"length":1},{"begin":598,"length":1},{"begin":600,"length":5},{"begin":606,"length":1},{"begin":608,"length":1},{"begin":611,"length":1},{"begin":619,"length":2},{"begin":623,"length":3},{"begin":629,"length":2},{"begin":632,"length":1},{"begin":634,"length":1},{"begin":636,"length":2},{"begin":640,"length":2},{"begin":644,"length":1},{"begin":648,"length":3},{"begin":652,"length":1},{"begin":657,"length":1},{"begin":659,"length":1},{"begin":671,"length":1},{"begin":673,"length":3},{"begin":678,"length":1},{"begin":685,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-06-08T07:31:40.987 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~39s , 2023-06-08T07:31:41.135 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json Ignoring malparsed backtrace: 2023-06-08T07:32:38.457 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":724,"fsid":"87dea55e-f8c6-48e7-a5de-497a76d690ca","created":"2023-06-08T07:19:03.434687+0000","modified":"2023-06-08T07:31:37.763740+0000","last_up_change":"2023-06-08T07:31:23.586612+0000","last_in_change":"2023-06-08T07:19:05.746653+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-06-08T07:19:11.856148+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-06-08T07:19:14.054099+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-06-08T07:19:21.041786+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"716","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":693,"snap_epoch":716,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"bc37cf1c-3e05-46a6-8dbc-b3e1a597b390","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":16,"down_at":711,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6800","nonce":2219628572}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6801","nonce":2219628572}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2219628572}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6805","nonce":2219628572}]},"public_addr":"172.21.15.99:6800/2219628572","cluster_addr":"172.21.15.99:6801/2219628572","heartbeat_back_addr":"172.21.15.99:6804/2219628572","heartbeat_front_addr":"172.21.15.99:6805/2219628572","state":["exists"]},{"osd":1,"uuid":"2628ebee-943b-4c37-b142-f4c7ab979a56","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":786871654}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6803","nonce":786871654}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6807","nonce":786871654}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":786871654}]},"public_addr":"172.21.15.99:6802/786871654","cluster_addr":"172.21.15.99:6803/786871654","heartbeat_back_addr":"172.21.15.99:6807/786871654","heartbeat_front_addr":"172.21.15.99:6806/786871654","state":["exists","up"]},{"osd":2,"uuid":"de6b6a32-e5a8-478f-8b75-94aebfb2b717","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6803","nonce":3055208471}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6802","nonce":3055208471}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6807","nonce":3055208471}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6808","nonce":3055208471}]},"public_addr":"172.21.15.162:6803/3055208471","cluster_addr":"172.21.15.162:6802/3055208471","heartbeat_back_addr":"172.21.15.162:6807/3055208471","heartbeat_front_addr":"172.21.15.162:6808/3055208471","state":["exists","up"]},{"osd":3,"uuid":"02b56dc3-6c99-49d7-b3d6-d5253d66b5c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6805","nonce":2372637129}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6804","nonce":2372637129}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6806","nonce":2372637129}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6809","nonce":2372637129}]},"public_addr":"172.21.15.162:6805/2372637129","cluster_addr":"172.21.15.162:6804/2372637129","heartbeat_back_addr":"172.21.15.162:6806/2372637129","heartbeat_front_addr":"172.21.15.162:6809/2372637129","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-06-08T07:31:23.586612+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":1},{"begin":8,"length":3},{"begin":14,"length":1},{"begin":16,"length":2},{"begin":20,"length":1},{"begin":22,"length":4},{"begin":27,"length":1},{"begin":31,"length":2},{"begin":36,"length":3},{"begin":40,"length":1},{"begin":42,"length":2},{"begin":45,"length":1},{"begin":49,"length":1},{"begin":51,"length":2},{"begin":55,"length":1},{"begin":57,"length":3},{"begin":61,"length":2},{"begin":64,"length":1},{"begin":68,"length":4},{"begin":75,"length":1},{"begin":77,"length":4},{"begin":82,"length":2},{"begin":86,"length":1},{"begin":89,"length":1},{"begin":91,"length":2},{"begin":94,"length":6},{"begin":101,"length":3},{"begin":106,"length":1},{"begin":108,"length":1},{"begin":111,"length":2},{"begin":114,"length":3},{"begin":118,"length":2},{"begin":121,"length":2},{"begin":124,"length":1},{"begin":135,"length":2},{"begin":138,"length":2},{"begin":142,"length":1},{"begin":145,"length":2},{"begin":148,"length":1},{"begin":150,"length":3},{"begin":160,"length":1},{"begin":162,"length":1},{"begin":165,"length":1},{"begin":169,"length":1},{"begin":171,"length":1},{"begin":173,"length":1},{"begin":176,"length":1},{"begin":178,"length":3},{"begin":184,"length":2},{"begin":188,"length":1},{"begin":190,"length":1},{"begin":192,"length":1},{"begin":194,"length":2},{"begin":198,"length":2},{"begin":202,"length":3},{"begin":208,"length":4},{"begin":215,"length":1},{"begin":218,"length":4},{"begin":226,"length":1},{"begin":228,"length":3},{"begin":233,"length":5},{"begin":240,"length":1},{"begin":243,"length":1},{"begin":246,"length":1},{"begin":248,"length":2},{"begin":251,"length":1},{"begin":253,"length":1},{"begin":256,"length":3},{"begin":263,"length":1},{"begin":266,"length":3},{"begin":270,"length":2},{"begin":274,"length":2},{"begin":277,"length":1},{"begin":280,"length":1},{"begin":284,"length":3},{"begin":288,"length":2},{"begin":291,"length":2},{"begin":295,"length":1},{"begin":297,"length":2},{"begin":303,"length":3},{"begin":307,"length":1},{"begin":309,"length":3},{"begin":313,"length":1},{"begin":316,"length":1},{"begin":319,"length":2},{"begin":323,"length":1},{"begin":326,"length":2},{"begin":329,"length":7},{"begin":338,"length":1},{"begin":342,"length":1},{"begin":345,"length":2},{"begin":348,"length":3},{"begin":353,"length":1},{"begin":357,"length":4},{"begin":363,"length":2},{"begin":369,"length":1},{"begin":373,"length":3},{"begin":377,"length":2},{"begin":380,"length":1},{"begin":382,"length":1},{"begin":385,"length":4},{"begin":393,"length":1},{"begin":396,"length":1},{"begin":401,"length":1},{"begin":403,"length":1},{"begin":405,"length":1},{"begin":407,"length":1},{"begin":409,"length":1},{"begin":411,"length":2},{"begin":415,"length":3},{"begin":420,"length":1},{"begin":422,"length":2},{"begin":425,"length":4},{"begin":432,"length":1},{"begin":438,"length":4},{"begin":443,"length":2},{"begin":446,"length":1},{"begin":450,"length":1},{"begin":454,"length":1},{"begin":456,"length":2},{"begin":460,"length":2},{"begin":463,"length":3},{"begin":467,"length":1},{"begin":469,"length":2},{"begin":473,"length":1},{"begin":478,"length":1},{"begin":480,"length":1},{"begin":482,"length":8},{"begin":492,"length":4},{"begin":497,"length":1},{"begin":499,"length":2},{"begin":502,"length":2},{"begin":506,"length":1},{"begin":509,"length":1},{"begin":512,"length":3},{"begin":517,"length":1},{"begin":520,"length":3},{"begin":529,"length":4},{"begin":534,"length":1},{"begin":537,"length":1},{"begin":539,"length":3},{"begin":543,"length":8},{"begin":552,"length":1},{"begin":555,"length":2},{"begin":558,"length":2},{"begin":562,"length":2},{"begin":565,"length":2},{"begin":568,"length":1},{"begin":572,"length":3},{"begin":576,"length":4},{"begin":581,"length":1},{"begin":583,"length":1},{"begin":587,"length":1},{"begin":594,"length":1},{"begin":598,"length":1},{"begin":600,"length":5},{"begin":606,"length":1},{"begin":608,"length":1},{"begin":611,"length":1},{"begin":619,"length":2},{"begin":623,"length":3},{"begin":629,"length":2},{"begin":632,"length":1},{"begin":634,"length":1},{"begin":636,"length":2},{"begin":640,"length":2},{"begin":644,"length":1},{"begin":648,"length":3},{"begin":652,"length":1},{"begin":657,"length":1},{"begin":659,"length":1},{"begin":671,"length":1},{"begin":673,"length":3},{"begin":678,"length":1},{"begin":685,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-06-08T07:32:41.473 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-06-08T07:32:41.544 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~99s Ignoring malparsed backtrace: 2023-06-08T07:33:39.028 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":724,"fsid":"87dea55e-f8c6-48e7-a5de-497a76d690ca","created":"2023-06-08T07:19:03.434687+0000","modified":"2023-06-08T07:31:37.763740+0000","last_up_change":"2023-06-08T07:31:23.586612+0000","last_in_change":"2023-06-08T07:19:05.746653+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-06-08T07:19:11.856148+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-06-08T07:19:14.054099+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-06-08T07:19:21.041786+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"716","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":693,"snap_epoch":716,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"bc37cf1c-3e05-46a6-8dbc-b3e1a597b390","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":16,"down_at":711,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6800","nonce":2219628572}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6801","nonce":2219628572}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2219628572}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6805","nonce":2219628572}]},"public_addr":"172.21.15.99:6800/2219628572","cluster_addr":"172.21.15.99:6801/2219628572","heartbeat_back_addr":"172.21.15.99:6804/2219628572","heartbeat_front_addr":"172.21.15.99:6805/2219628572","state":["exists"]},{"osd":1,"uuid":"2628ebee-943b-4c37-b142-f4c7ab979a56","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":786871654}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6803","nonce":786871654}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6807","nonce":786871654}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":786871654}]},"public_addr":"172.21.15.99:6802/786871654","cluster_addr":"172.21.15.99:6803/786871654","heartbeat_back_addr":"172.21.15.99:6807/786871654","heartbeat_front_addr":"172.21.15.99:6806/786871654","state":["exists","up"]},{"osd":2,"uuid":"de6b6a32-e5a8-478f-8b75-94aebfb2b717","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6803","nonce":3055208471}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6802","nonce":3055208471}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6807","nonce":3055208471}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6808","nonce":3055208471}]},"public_addr":"172.21.15.162:6803/3055208471","cluster_addr":"172.21.15.162:6802/3055208471","heartbeat_back_addr":"172.21.15.162:6807/3055208471","heartbeat_front_addr":"172.21.15.162:6808/3055208471","state":["exists","up"]},{"osd":3,"uuid":"02b56dc3-6c99-49d7-b3d6-d5253d66b5c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6805","nonce":2372637129}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6804","nonce":2372637129}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6806","nonce":2372637129}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6809","nonce":2372637129}]},"public_addr":"172.21.15.162:6805/2372637129","cluster_addr":"172.21.15.162:6804/2372637129","heartbeat_back_addr":"172.21.15.162:6806/2372637129","heartbeat_front_addr":"172.21.15.162:6809/2372637129","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-06-08T07:31:23.586612+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":1},{"begin":8,"length":3},{"begin":14,"length":1},{"begin":16,"length":2},{"begin":20,"length":1},{"begin":22,"length":4},{"begin":27,"length":1},{"begin":31,"length":2},{"begin":36,"length":3},{"begin":40,"length":1},{"begin":42,"length":2},{"begin":45,"length":1},{"begin":49,"length":1},{"begin":51,"length":2},{"begin":55,"length":1},{"begin":57,"length":3},{"begin":61,"length":2},{"begin":64,"length":1},{"begin":68,"length":4},{"begin":75,"length":1},{"begin":77,"length":4},{"begin":82,"length":2},{"begin":86,"length":1},{"begin":89,"length":1},{"begin":91,"length":2},{"begin":94,"length":6},{"begin":101,"length":3},{"begin":106,"length":1},{"begin":108,"length":1},{"begin":111,"length":2},{"begin":114,"length":3},{"begin":118,"length":2},{"begin":121,"length":2},{"begin":124,"length":1},{"begin":135,"length":2},{"begin":138,"length":2},{"begin":142,"length":1},{"begin":145,"length":2},{"begin":148,"length":1},{"begin":150,"length":3},{"begin":160,"length":1},{"begin":162,"length":1},{"begin":165,"length":1},{"begin":169,"length":1},{"begin":171,"length":1},{"begin":173,"length":1},{"begin":176,"length":1},{"begin":178,"length":3},{"begin":184,"length":2},{"begin":188,"length":1},{"begin":190,"length":1},{"begin":192,"length":1},{"begin":194,"length":2},{"begin":198,"length":2},{"begin":202,"length":3},{"begin":208,"length":4},{"begin":215,"length":1},{"begin":218,"length":4},{"begin":226,"length":1},{"begin":228,"length":3},{"begin":233,"length":5},{"begin":240,"length":1},{"begin":243,"length":1},{"begin":246,"length":1},{"begin":248,"length":2},{"begin":251,"length":1},{"begin":253,"length":1},{"begin":256,"length":3},{"begin":263,"length":1},{"begin":266,"length":3},{"begin":270,"length":2},{"begin":274,"length":2},{"begin":277,"length":1},{"begin":280,"length":1},{"begin":284,"length":3},{"begin":288,"length":2},{"begin":291,"length":2},{"begin":295,"length":1},{"begin":297,"length":2},{"begin":303,"length":3},{"begin":307,"length":1},{"begin":309,"length":3},{"begin":313,"length":1},{"begin":316,"length":1},{"begin":319,"length":2},{"begin":323,"length":1},{"begin":326,"length":2},{"begin":329,"length":7},{"begin":338,"length":1},{"begin":342,"length":1},{"begin":345,"length":2},{"begin":348,"length":3},{"begin":353,"length":1},{"begin":357,"length":4},{"begin":363,"length":2},{"begin":369,"length":1},{"begin":373,"length":3},{"begin":377,"length":2},{"begin":380,"length":1},{"begin":382,"length":1},{"begin":385,"length":4},{"begin":393,"length":1},{"begin":396,"length":1},{"begin":401,"length":1},{"begin":403,"length":1},{"begin":405,"length":1},{"begin":407,"length":1},{"begin":409,"length":1},{"begin":411,"length":2},{"begin":415,"length":3},{"begin":420,"length":1},{"begin":422,"length":2},{"begin":425,"length":4},{"begin":432,"length":1},{"begin":438,"length":4},{"begin":443,"length":2},{"begin":446,"length":1},{"begin":450,"length":1},{"begin":454,"length":1},{"begin":456,"length":2},{"begin":460,"length":2},{"begin":463,"length":3},{"begin":467,"length":1},{"begin":469,"length":2},{"begin":473,"length":1},{"begin":478,"length":1},{"begin":480,"length":1},{"begin":482,"length":8},{"begin":492,"length":4},{"begin":497,"length":1},{"begin":499,"length":2},{"begin":502,"length":2},{"begin":506,"length":1},{"begin":509,"length":1},{"begin":512,"length":3},{"begin":517,"length":1},{"begin":520,"length":3},{"begin":529,"length":4},{"begin":534,"length":1},{"begin":537,"length":1},{"begin":539,"length":3},{"begin":543,"length":8},{"begin":552,"length":1},{"begin":555,"length":2},{"begin":558,"length":2},{"begin":562,"length":2},{"begin":565,"length":2},{"begin":568,"length":1},{"begin":572,"length":3},{"begin":576,"length":4},{"begin":581,"length":1},{"begin":583,"length":1},{"begin":587,"length":1},{"begin":594,"length":1},{"begin":598,"length":1},{"begin":600,"length":5},{"begin":606,"length":1},{"begin":608,"length":1},{"begin":611,"length":1},{"begin":619,"length":2},{"begin":623,"length":3},{"begin":629,"length":2},{"begin":632,"length":1},{"begin":634,"length":1},{"begin":636,"length":2},{"begin":640,"length":2},{"begin":644,"length":1},{"begin":648,"length":3},{"begin":652,"length":1},{"begin":657,"length":1},{"begin":659,"length":1},{"begin":671,"length":1},{"begin":673,"length":3},{"begin":678,"length":1},{"begin":685,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-06-08T07:33:42.044 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-06-08T07:33:42.091 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~160s Ignoring malparsed backtrace: 2023-06-08T07:34:39.671 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":724,"fsid":"87dea55e-f8c6-48e7-a5de-497a76d690ca","created":"2023-06-08T07:19:03.434687+0000","modified":"2023-06-08T07:31:37.763740+0000","last_up_change":"2023-06-08T07:31:23.586612+0000","last_in_change":"2023-06-08T07:19:05.746653+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-06-08T07:19:11.856148+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-06-08T07:19:14.054099+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-06-08T07:19:21.041786+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"716","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":693,"snap_epoch":716,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"bc37cf1c-3e05-46a6-8dbc-b3e1a597b390","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":16,"down_at":711,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6800","nonce":2219628572}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6801","nonce":2219628572}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2219628572}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6805","nonce":2219628572}]},"public_addr":"172.21.15.99:6800/2219628572","cluster_addr":"172.21.15.99:6801/2219628572","heartbeat_back_addr":"172.21.15.99:6804/2219628572","heartbeat_front_addr":"172.21.15.99:6805/2219628572","state":["exists"]},{"osd":1,"uuid":"2628ebee-943b-4c37-b142-f4c7ab979a56","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":786871654}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6803","nonce":786871654}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6807","nonce":786871654}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":786871654}]},"public_addr":"172.21.15.99:6802/786871654","cluster_addr":"172.21.15.99:6803/786871654","heartbeat_back_addr":"172.21.15.99:6807/786871654","heartbeat_front_addr":"172.21.15.99:6806/786871654","state":["exists","up"]},{"osd":2,"uuid":"de6b6a32-e5a8-478f-8b75-94aebfb2b717","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6803","nonce":3055208471}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6802","nonce":3055208471}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6807","nonce":3055208471}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6808","nonce":3055208471}]},"public_addr":"172.21.15.162:6803/3055208471","cluster_addr":"172.21.15.162:6802/3055208471","heartbeat_back_addr":"172.21.15.162:6807/3055208471","heartbeat_front_addr":"172.21.15.162:6808/3055208471","state":["exists","up"]},{"osd":3,"uuid":"02b56dc3-6c99-49d7-b3d6-d5253d66b5c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6805","nonce":2372637129}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6804","nonce":2372637129}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6806","nonce":2372637129}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6809","nonce":2372637129}]},"public_addr":"172.21.15.162:6805/2372637129","cluster_addr":"172.21.15.162:6804/2372637129","heartbeat_back_addr":"172.21.15.162:6806/2372637129","heartbeat_front_addr":"172.21.15.162:6809/2372637129","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-06-08T07:31:23.586612+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":1},{"begin":8,"length":3},{"begin":14,"length":1},{"begin":16,"length":2},{"begin":20,"length":1},{"begin":22,"length":4},{"begin":27,"length":1},{"begin":31,"length":2},{"begin":36,"length":3},{"begin":40,"length":1},{"begin":42,"length":2},{"begin":45,"length":1},{"begin":49,"length":1},{"begin":51,"length":2},{"begin":55,"length":1},{"begin":57,"length":3},{"begin":61,"length":2},{"begin":64,"length":1},{"begin":68,"length":4},{"begin":75,"length":1},{"begin":77,"length":4},{"begin":82,"length":2},{"begin":86,"length":1},{"begin":89,"length":1},{"begin":91,"length":2},{"begin":94,"length":6},{"begin":101,"length":3},{"begin":106,"length":1},{"begin":108,"length":1},{"begin":111,"length":2},{"begin":114,"length":3},{"begin":118,"length":2},{"begin":121,"length":2},{"begin":124,"length":1},{"begin":135,"length":2},{"begin":138,"length":2},{"begin":142,"length":1},{"begin":145,"length":2},{"begin":148,"length":1},{"begin":150,"length":3},{"begin":160,"length":1},{"begin":162,"length":1},{"begin":165,"length":1},{"begin":169,"length":1},{"begin":171,"length":1},{"begin":173,"length":1},{"begin":176,"length":1},{"begin":178,"length":3},{"begin":184,"length":2},{"begin":188,"length":1},{"begin":190,"length":1},{"begin":192,"length":1},{"begin":194,"length":2},{"begin":198,"length":2},{"begin":202,"length":3},{"begin":208,"length":4},{"begin":215,"length":1},{"begin":218,"length":4},{"begin":226,"length":1},{"begin":228,"length":3},{"begin":233,"length":5},{"begin":240,"length":1},{"begin":243,"length":1},{"begin":246,"length":1},{"begin":248,"length":2},{"begin":251,"length":1},{"begin":253,"length":1},{"begin":256,"length":3},{"begin":263,"length":1},{"begin":266,"length":3},{"begin":270,"length":2},{"begin":274,"length":2},{"begin":277,"length":1},{"begin":280,"length":1},{"begin":284,"length":3},{"begin":288,"length":2},{"begin":291,"length":2},{"begin":295,"length":1},{"begin":297,"length":2},{"begin":303,"length":3},{"begin":307,"length":1},{"begin":309,"length":3},{"begin":313,"length":1},{"begin":316,"length":1},{"begin":319,"length":2},{"begin":323,"length":1},{"begin":326,"length":2},{"begin":329,"length":7},{"begin":338,"length":1},{"begin":342,"length":1},{"begin":345,"length":2},{"begin":348,"length":3},{"begin":353,"length":1},{"begin":357,"length":4},{"begin":363,"length":2},{"begin":369,"length":1},{"begin":373,"length":3},{"begin":377,"length":2},{"begin":380,"length":1},{"begin":382,"length":1},{"begin":385,"length":4},{"begin":393,"length":1},{"begin":396,"length":1},{"begin":401,"length":1},{"begin":403,"length":1},{"begin":405,"length":1},{"begin":407,"length":1},{"begin":409,"length":1},{"begin":411,"length":2},{"begin":415,"length":3},{"begin":420,"length":1},{"begin":422,"length":2},{"begin":425,"length":4},{"begin":432,"length":1},{"begin":438,"length":4},{"begin":443,"length":2},{"begin":446,"length":1},{"begin":450,"length":1},{"begin":454,"length":1},{"begin":456,"length":2},{"begin":460,"length":2},{"begin":463,"length":3},{"begin":467,"length":1},{"begin":469,"length":2},{"begin":473,"length":1},{"begin":478,"length":1},{"begin":480,"length":1},{"begin":482,"length":8},{"begin":492,"length":4},{"begin":497,"length":1},{"begin":499,"length":2},{"begin":502,"length":2},{"begin":506,"length":1},{"begin":509,"length":1},{"begin":512,"length":3},{"begin":517,"length":1},{"begin":520,"length":3},{"begin":529,"length":4},{"begin":534,"length":1},{"begin":537,"length":1},{"begin":539,"length":3},{"begin":543,"length":8},{"begin":552,"length":1},{"begin":555,"length":2},{"begin":558,"length":2},{"begin":562,"length":2},{"begin":565,"length":2},{"begin":568,"length":1},{"begin":572,"length":3},{"begin":576,"length":4},{"begin":581,"length":1},{"begin":583,"length":1},{"begin":587,"length":1},{"begin":594,"length":1},{"begin":598,"length":1},{"begin":600,"length":5},{"begin":606,"length":1},{"begin":608,"length":1},{"begin":611,"length":1},{"begin":619,"length":2},{"begin":623,"length":3},{"begin":629,"length":2},{"begin":632,"length":1},{"begin":634,"length":1},{"begin":636,"length":2},{"begin":640,"length":2},{"begin":644,"length":1},{"begin":648,"length":3},{"begin":652,"length":1},{"begin":657,"length":1},{"begin":659,"length":1},{"begin":671,"length":1},{"begin":673,"length":3},{"begin":678,"length":1},{"begin":685,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-06-08T07:34:42.638 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~220s , 2023-06-08T07:34:42.687 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json Ignoring malparsed backtrace: 2023-06-08T07:35:40.314 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":724,"fsid":"87dea55e-f8c6-48e7-a5de-497a76d690ca","created":"2023-06-08T07:19:03.434687+0000","modified":"2023-06-08T07:31:37.763740+0000","last_up_change":"2023-06-08T07:31:23.586612+0000","last_in_change":"2023-06-08T07:19:05.746653+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-06-08T07:19:11.856148+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-06-08T07:19:14.054099+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-06-08T07:19:21.041786+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"716","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":693,"snap_epoch":716,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"bc37cf1c-3e05-46a6-8dbc-b3e1a597b390","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":16,"down_at":711,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6800","nonce":2219628572}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6801","nonce":2219628572}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2219628572}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6805","nonce":2219628572}]},"public_addr":"172.21.15.99:6800/2219628572","cluster_addr":"172.21.15.99:6801/2219628572","heartbeat_back_addr":"172.21.15.99:6804/2219628572","heartbeat_front_addr":"172.21.15.99:6805/2219628572","state":["exists"]},{"osd":1,"uuid":"2628ebee-943b-4c37-b142-f4c7ab979a56","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":786871654}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6803","nonce":786871654}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6807","nonce":786871654}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":786871654}]},"public_addr":"172.21.15.99:6802/786871654","cluster_addr":"172.21.15.99:6803/786871654","heartbeat_back_addr":"172.21.15.99:6807/786871654","heartbeat_front_addr":"172.21.15.99:6806/786871654","state":["exists","up"]},{"osd":2,"uuid":"de6b6a32-e5a8-478f-8b75-94aebfb2b717","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6803","nonce":3055208471}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6802","nonce":3055208471}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6807","nonce":3055208471}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6808","nonce":3055208471}]},"public_addr":"172.21.15.162:6803/3055208471","cluster_addr":"172.21.15.162:6802/3055208471","heartbeat_back_addr":"172.21.15.162:6807/3055208471","heartbeat_front_addr":"172.21.15.162:6808/3055208471","state":["exists","up"]},{"osd":3,"uuid":"02b56dc3-6c99-49d7-b3d6-d5253d66b5c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6805","nonce":2372637129}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6804","nonce":2372637129}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6806","nonce":2372637129}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6809","nonce":2372637129}]},"public_addr":"172.21.15.162:6805/2372637129","cluster_addr":"172.21.15.162:6804/2372637129","heartbeat_back_addr":"172.21.15.162:6806/2372637129","heartbeat_front_addr":"172.21.15.162:6809/2372637129","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-06-08T07:31:23.586612+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":1},{"begin":8,"length":3},{"begin":14,"length":1},{"begin":16,"length":2},{"begin":20,"length":1},{"begin":22,"length":4},{"begin":27,"length":1},{"begin":31,"length":2},{"begin":36,"length":3},{"begin":40,"length":1},{"begin":42,"length":2},{"begin":45,"length":1},{"begin":49,"length":1},{"begin":51,"length":2},{"begin":55,"length":1},{"begin":57,"length":3},{"begin":61,"length":2},{"begin":64,"length":1},{"begin":68,"length":4},{"begin":75,"length":1},{"begin":77,"length":4},{"begin":82,"length":2},{"begin":86,"length":1},{"begin":89,"length":1},{"begin":91,"length":2},{"begin":94,"length":6},{"begin":101,"length":3},{"begin":106,"length":1},{"begin":108,"length":1},{"begin":111,"length":2},{"begin":114,"length":3},{"begin":118,"length":2},{"begin":121,"length":2},{"begin":124,"length":1},{"begin":135,"length":2},{"begin":138,"length":2},{"begin":142,"length":1},{"begin":145,"length":2},{"begin":148,"length":1},{"begin":150,"length":3},{"begin":160,"length":1},{"begin":162,"length":1},{"begin":165,"length":1},{"begin":169,"length":1},{"begin":171,"length":1},{"begin":173,"length":1},{"begin":176,"length":1},{"begin":178,"length":3},{"begin":184,"length":2},{"begin":188,"length":1},{"begin":190,"length":1},{"begin":192,"length":1},{"begin":194,"length":2},{"begin":198,"length":2},{"begin":202,"length":3},{"begin":208,"length":4},{"begin":215,"length":1},{"begin":218,"length":4},{"begin":226,"length":1},{"begin":228,"length":3},{"begin":233,"length":5},{"begin":240,"length":1},{"begin":243,"length":1},{"begin":246,"length":1},{"begin":248,"length":2},{"begin":251,"length":1},{"begin":253,"length":1},{"begin":256,"length":3},{"begin":263,"length":1},{"begin":266,"length":3},{"begin":270,"length":2},{"begin":274,"length":2},{"begin":277,"length":1},{"begin":280,"length":1},{"begin":284,"length":3},{"begin":288,"length":2},{"begin":291,"length":2},{"begin":295,"length":1},{"begin":297,"length":2},{"begin":303,"length":3},{"begin":307,"length":1},{"begin":309,"length":3},{"begin":313,"length":1},{"begin":316,"length":1},{"begin":319,"length":2},{"begin":323,"length":1},{"begin":326,"length":2},{"begin":329,"length":7},{"begin":338,"length":1},{"begin":342,"length":1},{"begin":345,"length":2},{"begin":348,"length":3},{"begin":353,"length":1},{"begin":357,"length":4},{"begin":363,"length":2},{"begin":369,"length":1},{"begin":373,"length":3},{"begin":377,"length":2},{"begin":380,"length":1},{"begin":382,"length":1},{"begin":385,"length":4},{"begin":393,"length":1},{"begin":396,"length":1},{"begin":401,"length":1},{"begin":403,"length":1},{"begin":405,"length":1},{"begin":407,"length":1},{"begin":409,"length":1},{"begin":411,"length":2},{"begin":415,"length":3},{"begin":420,"length":1},{"begin":422,"length":2},{"begin":425,"length":4},{"begin":432,"length":1},{"begin":438,"length":4},{"begin":443,"length":2},{"begin":446,"length":1},{"begin":450,"length":1},{"begin":454,"length":1},{"begin":456,"length":2},{"begin":460,"length":2},{"begin":463,"length":3},{"begin":467,"length":1},{"begin":469,"length":2},{"begin":473,"length":1},{"begin":478,"length":1},{"begin":480,"length":1},{"begin":482,"length":8},{"begin":492,"length":4},{"begin":497,"length":1},{"begin":499,"length":2},{"begin":502,"length":2},{"begin":506,"length":1},{"begin":509,"length":1},{"begin":512,"length":3},{"begin":517,"length":1},{"begin":520,"length":3},{"begin":529,"length":4},{"begin":534,"length":1},{"begin":537,"length":1},{"begin":539,"length":3},{"begin":543,"length":8},{"begin":552,"length":1},{"begin":555,"length":2},{"begin":558,"length":2},{"begin":562,"length":2},{"begin":565,"length":2},{"begin":568,"length":1},{"begin":572,"length":3},{"begin":576,"length":4},{"begin":581,"length":1},{"begin":583,"length":1},{"begin":587,"length":1},{"begin":594,"length":1},{"begin":598,"length":1},{"begin":600,"length":5},{"begin":606,"length":1},{"begin":608,"length":1},{"begin":611,"length":1},{"begin":619,"length":2},{"begin":623,"length":3},{"begin":629,"length":2},{"begin":632,"length":1},{"begin":634,"length":1},{"begin":636,"length":2},{"begin":640,"length":2},{"begin":644,"length":1},{"begin":648,"length":3},{"begin":652,"length":1},{"begin":657,"length":1},{"begin":659,"length":1},{"begin":671,"length":1},{"begin":673,"length":3},{"begin":678,"length":1},{"begin":685,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-06-08T07:35:43.185 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~281s , 2023-06-08T07:35:43.329 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json Ignoring malparsed backtrace: r = self._runner(client=self.ssh, name=self.shortname, **kwargs) , File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/orchestra/run.py", line 455, in run , r.wait() Ignoring malparsed backtrace: File "/home/teuthworker/src/github.com_ceph_ceph_ea807834105141b6938ce6a9dba28667420efa2e/qa/tasks/ceph.py", line 1469, in run_daemon , yield , File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/contextutil.py", line 32, in nested Ignoring malparsed backtrace: 2023-06-08T08:28:13.377 DEBUG:teuthology.orchestra.run.smithi099:> sync && sudo umount -f /var/lib/ceph/osd/ceph-1 , 2023-06-08T08:28:13.453 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-2 on ubuntu@smithi162.front.sepia.ceph.com , 2023-06-08T08:28:13.453 DEBUG:teuthology.orchestra.run.smithi162:> sync && sudo umount -f /var/lib/ceph/osd/ceph-2 Ignoring malparsed backtrace: /var/log/ceph/ceph-client.admin.127918.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.121150.log , 50.0% -- replaced with /var/log/ceph/ceph-client.admin.127918.log.gz , /var/log/ceph/ceph-client.admin.130308.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.123182.log Ignoring malparsed backtrace: 49.6% -- replaced with /var/log/ceph/ceph-client.admin.104245.log.gz , /var/log/ceph/ceph-client.admin.120435.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.117780.log , 50.0% -- replaced with /var/log/ceph/ceph-client.admin.120435.log.gz Ignoring malparsed backtrace: /var/log/ceph/ceph-client.admin.109652.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.124567.log , 50.0% -- replaced with /var/log/ceph/ceph-client.admin.109652.log.gz , /var/log/ceph/ceph-client.admin.104343.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.126564.log Ignoring malparsed backtrace: 50.0% -- replaced with /var/log/ceph/ceph-client.admin.128995.log.gz , /var/log/ceph/ceph-client.admin.122117.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.112523.log , 50.4% -- replaced with /var/log/ceph/ceph-client.admin.122117.log.gz Ignoring malparsed backtrace: /var/log/ceph/ceph-client.admin.125535.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.127679.log , 50.0% -- replaced with /var/log/ceph/ceph-client.admin.125535.log.gz , /var/log/ceph/ceph-client.admin.132894.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.103886.log Ignoring malparsed backtrace: /var/log/ceph/ceph-client.admin.111570.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.132032.log , 50.4% -- replaced with /var/log/ceph/ceph-client.admin.111570.log.gz , /var/log/ceph/ceph-client.admin.105643.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.135368.log Ignoring malparsed backtrace: 49.6% -- replaced with /var/log/ceph/ceph-client.admin.115634.log.gz , /var/log/ceph/ceph-client.admin.105211.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.110324.log , 50.0% -- replaced with /var/log/ceph/ceph-client.admin.105211.log.gz Ignoring malparsed backtrace: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.114661.log , /var/log/ceph/ceph-client.admin.103574.log: 50.0% -- replaced with /var/log/ceph/ceph-client.admin.103574.log.gz , /var/log/ceph/ceph-client.admin.133490.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.110814.log Ignoring malparsed backtrace: 50.4% -- replaced with /var/log/ceph/ceph-client.admin.112250.log.gz , /var/log/ceph/ceph-client.admin.120716.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.118263.log , 50.0% -- replaced with /var/log/ceph/ceph-client.admin.120716.log.gz Ignoring malparsed backtrace: 50.0% -- replaced with /var/log/ceph/ceph-client.admin.131038.log.gz , /var/log/ceph/ceph-client.admin.126772.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.111997.log , 50.0% -- replaced with /var/log/ceph/ceph-client.admin.126772.log.gz Ignoring malparsed backtrace: /var/log/ceph/ceph-client.admin.111786.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.132208.log , 49.2% -- replaced with /var/log/ceph/ceph-client.admin.111786.log.gz , /var/log/ceph/ceph-client.admin.113152.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.109491.log Ignoring malparsed backtrace: 50.0% -- replaced with /var/log/ceph/ceph-client.admin.114283.log.gz , /var/log/ceph/ceph-client.admin.125913.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.105037.log , 50.4% -- replaced with /var/log/ceph/ceph-client.admin.125913.log.gz Ignoring malparsed backtrace: sys 0m0.444s , 92.7% -- replaced with /var/log/ceph/ceph-osd.3.log.gz , Ignoring malparsed backtrace: 2023-06-08T08:28:21.213 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive , 2023-06-08T08:28:21.227 INFO:teuthology.task.internal:Transferring archived files... , 2023-06-08T08:28:21.229 DEBUG:teuthology.misc:Transferring archived files from smithi099:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/matan-2023-06-08_06:36:16-crimson-rados-wip-matanb-crimson-testing-bug-61504-distro-crimson-smithi/7298180/remote/smithi099 Ignoring malparsed backtrace: ceph version 18.0.0-4330-g9b3f3761 (9b3f3761ef331c3cedab822629c5de78b7d54292) reef (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7f07f68f8ed4] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x1b0) [0x5570926960ca] Ignoring malparsed backtrace: 2023-06-08T07:31:38.120 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":724,"fsid":"87dea55e-f8c6-48e7-a5de-497a76d690ca","created":"2023-06-08T07:19:03.434687+0000","modified":"2023-06-08T07:31:37.763740+0000","last_up_change":"2023-06-08T07:31:23.586612+0000","last_in_change":"2023-06-08T07:19:05.746653+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-06-08T07:19:11.856148+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-06-08T07:19:14.054099+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-06-08T07:19:21.041786+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"716","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":693,"snap_epoch":716,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"bc37cf1c-3e05-46a6-8dbc-b3e1a597b390","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":16,"down_at":711,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6800","nonce":2219628572}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6801","nonce":2219628572}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2219628572}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6805","nonce":2219628572}]},"public_addr":"172.21.15.99:6800/2219628572","cluster_addr":"172.21.15.99:6801/2219628572","heartbeat_back_addr":"172.21.15.99:6804/2219628572","heartbeat_front_addr":"172.21.15.99:6805/2219628572","state":["exists"]},{"osd":1,"uuid":"2628ebee-943b-4c37-b142-f4c7ab979a56","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":786871654}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6803","nonce":786871654}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6807","nonce":786871654}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":786871654}]},"public_addr":"172.21.15.99:6802/786871654","cluster_addr":"172.21.15.99:6803/786871654","heartbeat_back_addr":"172.21.15.99:6807/786871654","heartbeat_front_addr":"172.21.15.99:6806/786871654","state":["exists","up"]},{"osd":2,"uuid":"de6b6a32-e5a8-478f-8b75-94aebfb2b717","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6803","nonce":3055208471}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6802","nonce":3055208471}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6807","nonce":3055208471}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6808","nonce":3055208471}]},"public_addr":"172.21.15.162:6803/3055208471","cluster_addr":"172.21.15.162:6802/3055208471","heartbeat_back_addr":"172.21.15.162:6807/3055208471","heartbeat_front_addr":"172.21.15.162:6808/3055208471","state":["exists","up"]},{"osd":3,"uuid":"02b56dc3-6c99-49d7-b3d6-d5253d66b5c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6805","nonce":2372637129}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6804","nonce":2372637129}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6806","nonce":2372637129}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6809","nonce":2372637129}]},"public_addr":"172.21.15.162:6805/2372637129","cluster_addr":"172.21.15.162:6804/2372637129","heartbeat_back_addr":"172.21.15.162:6806/2372637129","heartbeat_front_addr":"172.21.15.162:6809/2372637129","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-06-08T07:31:23.586612+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":1},{"begin":8,"length":3},{"begin":14,"length":1},{"begin":16,"length":2},{"begin":20,"length":1},{"begin":22,"length":4},{"begin":27,"length":1},{"begin":31,"length":2},{"begin":36,"length":3},{"begin":40,"length":1},{"begin":42,"length":2},{"begin":45,"length":1},{"begin":49,"length":1},{"begin":51,"length":2},{"begin":55,"length":1},{"begin":57,"length":3},{"begin":61,"length":2},{"begin":64,"length":1},{"begin":68,"length":4},{"begin":75,"length":1},{"begin":77,"length":4},{"begin":82,"length":2},{"begin":86,"length":1},{"begin":89,"length":1},{"begin":91,"length":2},{"begin":94,"length":6},{"begin":101,"length":3},{"begin":106,"length":1},{"begin":108,"length":1},{"begin":111,"length":2},{"begin":114,"length":3},{"begin":118,"length":2},{"begin":121,"length":2},{"begin":124,"length":1},{"begin":135,"length":2},{"begin":138,"length":2},{"begin":142,"length":1},{"begin":145,"length":2},{"begin":148,"length":1},{"begin":150,"length":3},{"begin":160,"length":1},{"begin":162,"length":1},{"begin":165,"length":1},{"begin":169,"length":1},{"begin":171,"length":1},{"begin":173,"length":1},{"begin":176,"length":1},{"begin":178,"length":3},{"begin":184,"length":2},{"begin":188,"length":1},{"begin":190,"length":1},{"begin":192,"length":1},{"begin":194,"length":2},{"begin":198,"length":2},{"begin":202,"length":3},{"begin":208,"length":4},{"begin":215,"length":1},{"begin":218,"length":4},{"begin":226,"length":1},{"begin":228,"length":3},{"begin":233,"length":5},{"begin":240,"length":1},{"begin":243,"length":1},{"begin":246,"length":1},{"begin":248,"length":2},{"begin":251,"length":1},{"begin":253,"length":1},{"begin":256,"length":3},{"begin":263,"length":1},{"begin":266,"length":3},{"begin":270,"length":2},{"begin":274,"length":2},{"begin":277,"length":1},{"begin":280,"length":1},{"begin":284,"length":3},{"begin":288,"length":2},{"begin":291,"length":2},{"begin":295,"length":1},{"begin":297,"length":2},{"begin":303,"length":3},{"begin":307,"length":1},{"begin":309,"length":3},{"begin":313,"length":1},{"begin":316,"length":1},{"begin":319,"length":2},{"begin":323,"length":1},{"begin":326,"length":2},{"begin":329,"length":7},{"begin":338,"length":1},{"begin":342,"length":1},{"begin":345,"length":2},{"begin":348,"length":3},{"begin":353,"length":1},{"begin":357,"length":4},{"begin":363,"length":2},{"begin":369,"length":1},{"begin":373,"length":3},{"begin":377,"length":2},{"begin":380,"length":1},{"begin":382,"length":1},{"begin":385,"length":4},{"begin":393,"length":1},{"begin":396,"length":1},{"begin":401,"length":1},{"begin":403,"length":1},{"begin":405,"length":1},{"begin":407,"length":1},{"begin":409,"length":1},{"begin":411,"length":2},{"begin":415,"length":3},{"begin":420,"length":1},{"begin":422,"length":2},{"begin":425,"length":4},{"begin":432,"length":1},{"begin":438,"length":4},{"begin":443,"length":2},{"begin":446,"length":1},{"begin":450,"length":1},{"begin":454,"length":1},{"begin":456,"length":2},{"begin":460,"length":2},{"begin":463,"length":3},{"begin":467,"length":1},{"begin":469,"length":2},{"begin":473,"length":1},{"begin":478,"length":1},{"begin":480,"length":1},{"begin":482,"length":8},{"begin":492,"length":4},{"begin":497,"length":1},{"begin":499,"length":2},{"begin":502,"length":2},{"begin":506,"length":1},{"begin":509,"length":1},{"begin":512,"length":3},{"begin":517,"length":1},{"begin":520,"length":3},{"begin":529,"length":4},{"begin":534,"length":1},{"begin":537,"length":1},{"begin":539,"length":3},{"begin":543,"length":8},{"begin":552,"length":1},{"begin":555,"length":2},{"begin":558,"length":2},{"begin":562,"length":2},{"begin":565,"length":2},{"begin":568,"length":1},{"begin":572,"length":3},{"begin":576,"length":4},{"begin":581,"length":1},{"begin":583,"length":1},{"begin":587,"length":1},{"begin":594,"length":1},{"begin":598,"length":1},{"begin":600,"length":5},{"begin":606,"length":1},{"begin":608,"length":1},{"begin":611,"length":1},{"begin":619,"length":2},{"begin":623,"length":3},{"begin":629,"length":2},{"begin":632,"length":1},{"begin":634,"length":1},{"begin":636,"length":2},{"begin":640,"length":2},{"begin":644,"length":1},{"begin":648,"length":3},{"begin":652,"length":1},{"begin":657,"length":1},{"begin":659,"length":1},{"begin":671,"length":1},{"begin":673,"length":3},{"begin":678,"length":1},{"begin":685,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-06-08T07:31:40.987 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~39s , 2023-06-08T07:31:41.135 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json Ignoring malparsed backtrace: 2023-06-08T07:32:38.457 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":724,"fsid":"87dea55e-f8c6-48e7-a5de-497a76d690ca","created":"2023-06-08T07:19:03.434687+0000","modified":"2023-06-08T07:31:37.763740+0000","last_up_change":"2023-06-08T07:31:23.586612+0000","last_in_change":"2023-06-08T07:19:05.746653+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-06-08T07:19:11.856148+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-06-08T07:19:14.054099+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-06-08T07:19:21.041786+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"716","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":693,"snap_epoch":716,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"bc37cf1c-3e05-46a6-8dbc-b3e1a597b390","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":16,"down_at":711,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6800","nonce":2219628572}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6801","nonce":2219628572}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2219628572}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6805","nonce":2219628572}]},"public_addr":"172.21.15.99:6800/2219628572","cluster_addr":"172.21.15.99:6801/2219628572","heartbeat_back_addr":"172.21.15.99:6804/2219628572","heartbeat_front_addr":"172.21.15.99:6805/2219628572","state":["exists"]},{"osd":1,"uuid":"2628ebee-943b-4c37-b142-f4c7ab979a56","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":786871654}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6803","nonce":786871654}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6807","nonce":786871654}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":786871654}]},"public_addr":"172.21.15.99:6802/786871654","cluster_addr":"172.21.15.99:6803/786871654","heartbeat_back_addr":"172.21.15.99:6807/786871654","heartbeat_front_addr":"172.21.15.99:6806/786871654","state":["exists","up"]},{"osd":2,"uuid":"de6b6a32-e5a8-478f-8b75-94aebfb2b717","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6803","nonce":3055208471}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6802","nonce":3055208471}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6807","nonce":3055208471}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6808","nonce":3055208471}]},"public_addr":"172.21.15.162:6803/3055208471","cluster_addr":"172.21.15.162:6802/3055208471","heartbeat_back_addr":"172.21.15.162:6807/3055208471","heartbeat_front_addr":"172.21.15.162:6808/3055208471","state":["exists","up"]},{"osd":3,"uuid":"02b56dc3-6c99-49d7-b3d6-d5253d66b5c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6805","nonce":2372637129}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6804","nonce":2372637129}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6806","nonce":2372637129}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6809","nonce":2372637129}]},"public_addr":"172.21.15.162:6805/2372637129","cluster_addr":"172.21.15.162:6804/2372637129","heartbeat_back_addr":"172.21.15.162:6806/2372637129","heartbeat_front_addr":"172.21.15.162:6809/2372637129","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-06-08T07:31:23.586612+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":1},{"begin":8,"length":3},{"begin":14,"length":1},{"begin":16,"length":2},{"begin":20,"length":1},{"begin":22,"length":4},{"begin":27,"length":1},{"begin":31,"length":2},{"begin":36,"length":3},{"begin":40,"length":1},{"begin":42,"length":2},{"begin":45,"length":1},{"begin":49,"length":1},{"begin":51,"length":2},{"begin":55,"length":1},{"begin":57,"length":3},{"begin":61,"length":2},{"begin":64,"length":1},{"begin":68,"length":4},{"begin":75,"length":1},{"begin":77,"length":4},{"begin":82,"length":2},{"begin":86,"length":1},{"begin":89,"length":1},{"begin":91,"length":2},{"begin":94,"length":6},{"begin":101,"length":3},{"begin":106,"length":1},{"begin":108,"length":1},{"begin":111,"length":2},{"begin":114,"length":3},{"begin":118,"length":2},{"begin":121,"length":2},{"begin":124,"length":1},{"begin":135,"length":2},{"begin":138,"length":2},{"begin":142,"length":1},{"begin":145,"length":2},{"begin":148,"length":1},{"begin":150,"length":3},{"begin":160,"length":1},{"begin":162,"length":1},{"begin":165,"length":1},{"begin":169,"length":1},{"begin":171,"length":1},{"begin":173,"length":1},{"begin":176,"length":1},{"begin":178,"length":3},{"begin":184,"length":2},{"begin":188,"length":1},{"begin":190,"length":1},{"begin":192,"length":1},{"begin":194,"length":2},{"begin":198,"length":2},{"begin":202,"length":3},{"begin":208,"length":4},{"begin":215,"length":1},{"begin":218,"length":4},{"begin":226,"length":1},{"begin":228,"length":3},{"begin":233,"length":5},{"begin":240,"length":1},{"begin":243,"length":1},{"begin":246,"length":1},{"begin":248,"length":2},{"begin":251,"length":1},{"begin":253,"length":1},{"begin":256,"length":3},{"begin":263,"length":1},{"begin":266,"length":3},{"begin":270,"length":2},{"begin":274,"length":2},{"begin":277,"length":1},{"begin":280,"length":1},{"begin":284,"length":3},{"begin":288,"length":2},{"begin":291,"length":2},{"begin":295,"length":1},{"begin":297,"length":2},{"begin":303,"length":3},{"begin":307,"length":1},{"begin":309,"length":3},{"begin":313,"length":1},{"begin":316,"length":1},{"begin":319,"length":2},{"begin":323,"length":1},{"begin":326,"length":2},{"begin":329,"length":7},{"begin":338,"length":1},{"begin":342,"length":1},{"begin":345,"length":2},{"begin":348,"length":3},{"begin":353,"length":1},{"begin":357,"length":4},{"begin":363,"length":2},{"begin":369,"length":1},{"begin":373,"length":3},{"begin":377,"length":2},{"begin":380,"length":1},{"begin":382,"length":1},{"begin":385,"length":4},{"begin":393,"length":1},{"begin":396,"length":1},{"begin":401,"length":1},{"begin":403,"length":1},{"begin":405,"length":1},{"begin":407,"length":1},{"begin":409,"length":1},{"begin":411,"length":2},{"begin":415,"length":3},{"begin":420,"length":1},{"begin":422,"length":2},{"begin":425,"length":4},{"begin":432,"length":1},{"begin":438,"length":4},{"begin":443,"length":2},{"begin":446,"length":1},{"begin":450,"length":1},{"begin":454,"length":1},{"begin":456,"length":2},{"begin":460,"length":2},{"begin":463,"length":3},{"begin":467,"length":1},{"begin":469,"length":2},{"begin":473,"length":1},{"begin":478,"length":1},{"begin":480,"length":1},{"begin":482,"length":8},{"begin":492,"length":4},{"begin":497,"length":1},{"begin":499,"length":2},{"begin":502,"length":2},{"begin":506,"length":1},{"begin":509,"length":1},{"begin":512,"length":3},{"begin":517,"length":1},{"begin":520,"length":3},{"begin":529,"length":4},{"begin":534,"length":1},{"begin":537,"length":1},{"begin":539,"length":3},{"begin":543,"length":8},{"begin":552,"length":1},{"begin":555,"length":2},{"begin":558,"length":2},{"begin":562,"length":2},{"begin":565,"length":2},{"begin":568,"length":1},{"begin":572,"length":3},{"begin":576,"length":4},{"begin":581,"length":1},{"begin":583,"length":1},{"begin":587,"length":1},{"begin":594,"length":1},{"begin":598,"length":1},{"begin":600,"length":5},{"begin":606,"length":1},{"begin":608,"length":1},{"begin":611,"length":1},{"begin":619,"length":2},{"begin":623,"length":3},{"begin":629,"length":2},{"begin":632,"length":1},{"begin":634,"length":1},{"begin":636,"length":2},{"begin":640,"length":2},{"begin":644,"length":1},{"begin":648,"length":3},{"begin":652,"length":1},{"begin":657,"length":1},{"begin":659,"length":1},{"begin":671,"length":1},{"begin":673,"length":3},{"begin":678,"length":1},{"begin":685,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-06-08T07:32:41.473 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-06-08T07:32:41.544 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~99s Ignoring malparsed backtrace: 2023-06-08T07:33:39.028 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":724,"fsid":"87dea55e-f8c6-48e7-a5de-497a76d690ca","created":"2023-06-08T07:19:03.434687+0000","modified":"2023-06-08T07:31:37.763740+0000","last_up_change":"2023-06-08T07:31:23.586612+0000","last_in_change":"2023-06-08T07:19:05.746653+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-06-08T07:19:11.856148+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-06-08T07:19:14.054099+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-06-08T07:19:21.041786+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"716","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":693,"snap_epoch":716,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"bc37cf1c-3e05-46a6-8dbc-b3e1a597b390","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":16,"down_at":711,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6800","nonce":2219628572}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6801","nonce":2219628572}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2219628572}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6805","nonce":2219628572}]},"public_addr":"172.21.15.99:6800/2219628572","cluster_addr":"172.21.15.99:6801/2219628572","heartbeat_back_addr":"172.21.15.99:6804/2219628572","heartbeat_front_addr":"172.21.15.99:6805/2219628572","state":["exists"]},{"osd":1,"uuid":"2628ebee-943b-4c37-b142-f4c7ab979a56","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":786871654}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6803","nonce":786871654}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6807","nonce":786871654}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":786871654}]},"public_addr":"172.21.15.99:6802/786871654","cluster_addr":"172.21.15.99:6803/786871654","heartbeat_back_addr":"172.21.15.99:6807/786871654","heartbeat_front_addr":"172.21.15.99:6806/786871654","state":["exists","up"]},{"osd":2,"uuid":"de6b6a32-e5a8-478f-8b75-94aebfb2b717","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6803","nonce":3055208471}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6802","nonce":3055208471}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6807","nonce":3055208471}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6808","nonce":3055208471}]},"public_addr":"172.21.15.162:6803/3055208471","cluster_addr":"172.21.15.162:6802/3055208471","heartbeat_back_addr":"172.21.15.162:6807/3055208471","heartbeat_front_addr":"172.21.15.162:6808/3055208471","state":["exists","up"]},{"osd":3,"uuid":"02b56dc3-6c99-49d7-b3d6-d5253d66b5c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6805","nonce":2372637129}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6804","nonce":2372637129}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6806","nonce":2372637129}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6809","nonce":2372637129}]},"public_addr":"172.21.15.162:6805/2372637129","cluster_addr":"172.21.15.162:6804/2372637129","heartbeat_back_addr":"172.21.15.162:6806/2372637129","heartbeat_front_addr":"172.21.15.162:6809/2372637129","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-06-08T07:31:23.586612+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":1},{"begin":8,"length":3},{"begin":14,"length":1},{"begin":16,"length":2},{"begin":20,"length":1},{"begin":22,"length":4},{"begin":27,"length":1},{"begin":31,"length":2},{"begin":36,"length":3},{"begin":40,"length":1},{"begin":42,"length":2},{"begin":45,"length":1},{"begin":49,"length":1},{"begin":51,"length":2},{"begin":55,"length":1},{"begin":57,"length":3},{"begin":61,"length":2},{"begin":64,"length":1},{"begin":68,"length":4},{"begin":75,"length":1},{"begin":77,"length":4},{"begin":82,"length":2},{"begin":86,"length":1},{"begin":89,"length":1},{"begin":91,"length":2},{"begin":94,"length":6},{"begin":101,"length":3},{"begin":106,"length":1},{"begin":108,"length":1},{"begin":111,"length":2},{"begin":114,"length":3},{"begin":118,"length":2},{"begin":121,"length":2},{"begin":124,"length":1},{"begin":135,"length":2},{"begin":138,"length":2},{"begin":142,"length":1},{"begin":145,"length":2},{"begin":148,"length":1},{"begin":150,"length":3},{"begin":160,"length":1},{"begin":162,"length":1},{"begin":165,"length":1},{"begin":169,"length":1},{"begin":171,"length":1},{"begin":173,"length":1},{"begin":176,"length":1},{"begin":178,"length":3},{"begin":184,"length":2},{"begin":188,"length":1},{"begin":190,"length":1},{"begin":192,"length":1},{"begin":194,"length":2},{"begin":198,"length":2},{"begin":202,"length":3},{"begin":208,"length":4},{"begin":215,"length":1},{"begin":218,"length":4},{"begin":226,"length":1},{"begin":228,"length":3},{"begin":233,"length":5},{"begin":240,"length":1},{"begin":243,"length":1},{"begin":246,"length":1},{"begin":248,"length":2},{"begin":251,"length":1},{"begin":253,"length":1},{"begin":256,"length":3},{"begin":263,"length":1},{"begin":266,"length":3},{"begin":270,"length":2},{"begin":274,"length":2},{"begin":277,"length":1},{"begin":280,"length":1},{"begin":284,"length":3},{"begin":288,"length":2},{"begin":291,"length":2},{"begin":295,"length":1},{"begin":297,"length":2},{"begin":303,"length":3},{"begin":307,"length":1},{"begin":309,"length":3},{"begin":313,"length":1},{"begin":316,"length":1},{"begin":319,"length":2},{"begin":323,"length":1},{"begin":326,"length":2},{"begin":329,"length":7},{"begin":338,"length":1},{"begin":342,"length":1},{"begin":345,"length":2},{"begin":348,"length":3},{"begin":353,"length":1},{"begin":357,"length":4},{"begin":363,"length":2},{"begin":369,"length":1},{"begin":373,"length":3},{"begin":377,"length":2},{"begin":380,"length":1},{"begin":382,"length":1},{"begin":385,"length":4},{"begin":393,"length":1},{"begin":396,"length":1},{"begin":401,"length":1},{"begin":403,"length":1},{"begin":405,"length":1},{"begin":407,"length":1},{"begin":409,"length":1},{"begin":411,"length":2},{"begin":415,"length":3},{"begin":420,"length":1},{"begin":422,"length":2},{"begin":425,"length":4},{"begin":432,"length":1},{"begin":438,"length":4},{"begin":443,"length":2},{"begin":446,"length":1},{"begin":450,"length":1},{"begin":454,"length":1},{"begin":456,"length":2},{"begin":460,"length":2},{"begin":463,"length":3},{"begin":467,"length":1},{"begin":469,"length":2},{"begin":473,"length":1},{"begin":478,"length":1},{"begin":480,"length":1},{"begin":482,"length":8},{"begin":492,"length":4},{"begin":497,"length":1},{"begin":499,"length":2},{"begin":502,"length":2},{"begin":506,"length":1},{"begin":509,"length":1},{"begin":512,"length":3},{"begin":517,"length":1},{"begin":520,"length":3},{"begin":529,"length":4},{"begin":534,"length":1},{"begin":537,"length":1},{"begin":539,"length":3},{"begin":543,"length":8},{"begin":552,"length":1},{"begin":555,"length":2},{"begin":558,"length":2},{"begin":562,"length":2},{"begin":565,"length":2},{"begin":568,"length":1},{"begin":572,"length":3},{"begin":576,"length":4},{"begin":581,"length":1},{"begin":583,"length":1},{"begin":587,"length":1},{"begin":594,"length":1},{"begin":598,"length":1},{"begin":600,"length":5},{"begin":606,"length":1},{"begin":608,"length":1},{"begin":611,"length":1},{"begin":619,"length":2},{"begin":623,"length":3},{"begin":629,"length":2},{"begin":632,"length":1},{"begin":634,"length":1},{"begin":636,"length":2},{"begin":640,"length":2},{"begin":644,"length":1},{"begin":648,"length":3},{"begin":652,"length":1},{"begin":657,"length":1},{"begin":659,"length":1},{"begin":671,"length":1},{"begin":673,"length":3},{"begin":678,"length":1},{"begin":685,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-06-08T07:33:42.044 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-06-08T07:33:42.091 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~160s Ignoring malparsed backtrace: 2023-06-08T07:34:39.671 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":724,"fsid":"87dea55e-f8c6-48e7-a5de-497a76d690ca","created":"2023-06-08T07:19:03.434687+0000","modified":"2023-06-08T07:31:37.763740+0000","last_up_change":"2023-06-08T07:31:23.586612+0000","last_in_change":"2023-06-08T07:19:05.746653+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-06-08T07:19:11.856148+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-06-08T07:19:14.054099+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-06-08T07:19:21.041786+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"716","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":693,"snap_epoch":716,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"bc37cf1c-3e05-46a6-8dbc-b3e1a597b390","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":16,"down_at":711,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6800","nonce":2219628572}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6801","nonce":2219628572}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2219628572}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6805","nonce":2219628572}]},"public_addr":"172.21.15.99:6800/2219628572","cluster_addr":"172.21.15.99:6801/2219628572","heartbeat_back_addr":"172.21.15.99:6804/2219628572","heartbeat_front_addr":"172.21.15.99:6805/2219628572","state":["exists"]},{"osd":1,"uuid":"2628ebee-943b-4c37-b142-f4c7ab979a56","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":786871654}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6803","nonce":786871654}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6807","nonce":786871654}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":786871654}]},"public_addr":"172.21.15.99:6802/786871654","cluster_addr":"172.21.15.99:6803/786871654","heartbeat_back_addr":"172.21.15.99:6807/786871654","heartbeat_front_addr":"172.21.15.99:6806/786871654","state":["exists","up"]},{"osd":2,"uuid":"de6b6a32-e5a8-478f-8b75-94aebfb2b717","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6803","nonce":3055208471}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6802","nonce":3055208471}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6807","nonce":3055208471}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6808","nonce":3055208471}]},"public_addr":"172.21.15.162:6803/3055208471","cluster_addr":"172.21.15.162:6802/3055208471","heartbeat_back_addr":"172.21.15.162:6807/3055208471","heartbeat_front_addr":"172.21.15.162:6808/3055208471","state":["exists","up"]},{"osd":3,"uuid":"02b56dc3-6c99-49d7-b3d6-d5253d66b5c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6805","nonce":2372637129}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6804","nonce":2372637129}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6806","nonce":2372637129}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6809","nonce":2372637129}]},"public_addr":"172.21.15.162:6805/2372637129","cluster_addr":"172.21.15.162:6804/2372637129","heartbeat_back_addr":"172.21.15.162:6806/2372637129","heartbeat_front_addr":"172.21.15.162:6809/2372637129","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-06-08T07:31:23.586612+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":1},{"begin":8,"length":3},{"begin":14,"length":1},{"begin":16,"length":2},{"begin":20,"length":1},{"begin":22,"length":4},{"begin":27,"length":1},{"begin":31,"length":2},{"begin":36,"length":3},{"begin":40,"length":1},{"begin":42,"length":2},{"begin":45,"length":1},{"begin":49,"length":1},{"begin":51,"length":2},{"begin":55,"length":1},{"begin":57,"length":3},{"begin":61,"length":2},{"begin":64,"length":1},{"begin":68,"length":4},{"begin":75,"length":1},{"begin":77,"length":4},{"begin":82,"length":2},{"begin":86,"length":1},{"begin":89,"length":1},{"begin":91,"length":2},{"begin":94,"length":6},{"begin":101,"length":3},{"begin":106,"length":1},{"begin":108,"length":1},{"begin":111,"length":2},{"begin":114,"length":3},{"begin":118,"length":2},{"begin":121,"length":2},{"begin":124,"length":1},{"begin":135,"length":2},{"begin":138,"length":2},{"begin":142,"length":1},{"begin":145,"length":2},{"begin":148,"length":1},{"begin":150,"length":3},{"begin":160,"length":1},{"begin":162,"length":1},{"begin":165,"length":1},{"begin":169,"length":1},{"begin":171,"length":1},{"begin":173,"length":1},{"begin":176,"length":1},{"begin":178,"length":3},{"begin":184,"length":2},{"begin":188,"length":1},{"begin":190,"length":1},{"begin":192,"length":1},{"begin":194,"length":2},{"begin":198,"length":2},{"begin":202,"length":3},{"begin":208,"length":4},{"begin":215,"length":1},{"begin":218,"length":4},{"begin":226,"length":1},{"begin":228,"length":3},{"begin":233,"length":5},{"begin":240,"length":1},{"begin":243,"length":1},{"begin":246,"length":1},{"begin":248,"length":2},{"begin":251,"length":1},{"begin":253,"length":1},{"begin":256,"length":3},{"begin":263,"length":1},{"begin":266,"length":3},{"begin":270,"length":2},{"begin":274,"length":2},{"begin":277,"length":1},{"begin":280,"length":1},{"begin":284,"length":3},{"begin":288,"length":2},{"begin":291,"length":2},{"begin":295,"length":1},{"begin":297,"length":2},{"begin":303,"length":3},{"begin":307,"length":1},{"begin":309,"length":3},{"begin":313,"length":1},{"begin":316,"length":1},{"begin":319,"length":2},{"begin":323,"length":1},{"begin":326,"length":2},{"begin":329,"length":7},{"begin":338,"length":1},{"begin":342,"length":1},{"begin":345,"length":2},{"begin":348,"length":3},{"begin":353,"length":1},{"begin":357,"length":4},{"begin":363,"length":2},{"begin":369,"length":1},{"begin":373,"length":3},{"begin":377,"length":2},{"begin":380,"length":1},{"begin":382,"length":1},{"begin":385,"length":4},{"begin":393,"length":1},{"begin":396,"length":1},{"begin":401,"length":1},{"begin":403,"length":1},{"begin":405,"length":1},{"begin":407,"length":1},{"begin":409,"length":1},{"begin":411,"length":2},{"begin":415,"length":3},{"begin":420,"length":1},{"begin":422,"length":2},{"begin":425,"length":4},{"begin":432,"length":1},{"begin":438,"length":4},{"begin":443,"length":2},{"begin":446,"length":1},{"begin":450,"length":1},{"begin":454,"length":1},{"begin":456,"length":2},{"begin":460,"length":2},{"begin":463,"length":3},{"begin":467,"length":1},{"begin":469,"length":2},{"begin":473,"length":1},{"begin":478,"length":1},{"begin":480,"length":1},{"begin":482,"length":8},{"begin":492,"length":4},{"begin":497,"length":1},{"begin":499,"length":2},{"begin":502,"length":2},{"begin":506,"length":1},{"begin":509,"length":1},{"begin":512,"length":3},{"begin":517,"length":1},{"begin":520,"length":3},{"begin":529,"length":4},{"begin":534,"length":1},{"begin":537,"length":1},{"begin":539,"length":3},{"begin":543,"length":8},{"begin":552,"length":1},{"begin":555,"length":2},{"begin":558,"length":2},{"begin":562,"length":2},{"begin":565,"length":2},{"begin":568,"length":1},{"begin":572,"length":3},{"begin":576,"length":4},{"begin":581,"length":1},{"begin":583,"length":1},{"begin":587,"length":1},{"begin":594,"length":1},{"begin":598,"length":1},{"begin":600,"length":5},{"begin":606,"length":1},{"begin":608,"length":1},{"begin":611,"length":1},{"begin":619,"length":2},{"begin":623,"length":3},{"begin":629,"length":2},{"begin":632,"length":1},{"begin":634,"length":1},{"begin":636,"length":2},{"begin":640,"length":2},{"begin":644,"length":1},{"begin":648,"length":3},{"begin":652,"length":1},{"begin":657,"length":1},{"begin":659,"length":1},{"begin":671,"length":1},{"begin":673,"length":3},{"begin":678,"length":1},{"begin":685,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-06-08T07:34:42.638 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~220s , 2023-06-08T07:34:42.687 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json Ignoring malparsed backtrace: 2023-06-08T07:35:40.314 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":724,"fsid":"87dea55e-f8c6-48e7-a5de-497a76d690ca","created":"2023-06-08T07:19:03.434687+0000","modified":"2023-06-08T07:31:37.763740+0000","last_up_change":"2023-06-08T07:31:23.586612+0000","last_in_change":"2023-06-08T07:19:05.746653+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-06-08T07:19:11.856148+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-06-08T07:19:14.054099+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-06-08T07:19:21.041786+0000","flags":270369,"flags_names":"hashpspool,nopgchange,selfmanaged_snaps,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"716","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":693,"snap_epoch":716,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1.75,"score_stable":1.75,"optimal_score":0.75,"raw_score_acting":1.309999942779541,"raw_score_stable":1.309999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"bc37cf1c-3e05-46a6-8dbc-b3e1a597b390","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":16,"down_at":711,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6800","nonce":2219628572}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6801","nonce":2219628572}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2219628572}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6805","nonce":2219628572}]},"public_addr":"172.21.15.99:6800/2219628572","cluster_addr":"172.21.15.99:6801/2219628572","heartbeat_back_addr":"172.21.15.99:6804/2219628572","heartbeat_front_addr":"172.21.15.99:6805/2219628572","state":["exists"]},{"osd":1,"uuid":"2628ebee-943b-4c37-b142-f4c7ab979a56","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":786871654}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6803","nonce":786871654}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6807","nonce":786871654}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":786871654}]},"public_addr":"172.21.15.99:6802/786871654","cluster_addr":"172.21.15.99:6803/786871654","heartbeat_back_addr":"172.21.15.99:6807/786871654","heartbeat_front_addr":"172.21.15.99:6806/786871654","state":["exists","up"]},{"osd":2,"uuid":"de6b6a32-e5a8-478f-8b75-94aebfb2b717","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6803","nonce":3055208471}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6802","nonce":3055208471}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6807","nonce":3055208471}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6808","nonce":3055208471}]},"public_addr":"172.21.15.162:6803/3055208471","cluster_addr":"172.21.15.162:6802/3055208471","heartbeat_back_addr":"172.21.15.162:6807/3055208471","heartbeat_front_addr":"172.21.15.162:6808/3055208471","state":["exists","up"]},{"osd":3,"uuid":"02b56dc3-6c99-49d7-b3d6-d5253d66b5c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":711,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6805","nonce":2372637129}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6804","nonce":2372637129}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6806","nonce":2372637129}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6809","nonce":2372637129}]},"public_addr":"172.21.15.162:6805/2372637129","cluster_addr":"172.21.15.162:6804/2372637129","heartbeat_back_addr":"172.21.15.162:6806/2372637129","heartbeat_front_addr":"172.21.15.162:6809/2372637129","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-06-08T07:31:23.586612+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":2,"length":2},{"begin":5,"length":1},{"begin":8,"length":3},{"begin":14,"length":1},{"begin":16,"length":2},{"begin":20,"length":1},{"begin":22,"length":4},{"begin":27,"length":1},{"begin":31,"length":2},{"begin":36,"length":3},{"begin":40,"length":1},{"begin":42,"length":2},{"begin":45,"length":1},{"begin":49,"length":1},{"begin":51,"length":2},{"begin":55,"length":1},{"begin":57,"length":3},{"begin":61,"length":2},{"begin":64,"length":1},{"begin":68,"length":4},{"begin":75,"length":1},{"begin":77,"length":4},{"begin":82,"length":2},{"begin":86,"length":1},{"begin":89,"length":1},{"begin":91,"length":2},{"begin":94,"length":6},{"begin":101,"length":3},{"begin":106,"length":1},{"begin":108,"length":1},{"begin":111,"length":2},{"begin":114,"length":3},{"begin":118,"length":2},{"begin":121,"length":2},{"begin":124,"length":1},{"begin":135,"length":2},{"begin":138,"length":2},{"begin":142,"length":1},{"begin":145,"length":2},{"begin":148,"length":1},{"begin":150,"length":3},{"begin":160,"length":1},{"begin":162,"length":1},{"begin":165,"length":1},{"begin":169,"length":1},{"begin":171,"length":1},{"begin":173,"length":1},{"begin":176,"length":1},{"begin":178,"length":3},{"begin":184,"length":2},{"begin":188,"length":1},{"begin":190,"length":1},{"begin":192,"length":1},{"begin":194,"length":2},{"begin":198,"length":2},{"begin":202,"length":3},{"begin":208,"length":4},{"begin":215,"length":1},{"begin":218,"length":4},{"begin":226,"length":1},{"begin":228,"length":3},{"begin":233,"length":5},{"begin":240,"length":1},{"begin":243,"length":1},{"begin":246,"length":1},{"begin":248,"length":2},{"begin":251,"length":1},{"begin":253,"length":1},{"begin":256,"length":3},{"begin":263,"length":1},{"begin":266,"length":3},{"begin":270,"length":2},{"begin":274,"length":2},{"begin":277,"length":1},{"begin":280,"length":1},{"begin":284,"length":3},{"begin":288,"length":2},{"begin":291,"length":2},{"begin":295,"length":1},{"begin":297,"length":2},{"begin":303,"length":3},{"begin":307,"length":1},{"begin":309,"length":3},{"begin":313,"length":1},{"begin":316,"length":1},{"begin":319,"length":2},{"begin":323,"length":1},{"begin":326,"length":2},{"begin":329,"length":7},{"begin":338,"length":1},{"begin":342,"length":1},{"begin":345,"length":2},{"begin":348,"length":3},{"begin":353,"length":1},{"begin":357,"length":4},{"begin":363,"length":2},{"begin":369,"length":1},{"begin":373,"length":3},{"begin":377,"length":2},{"begin":380,"length":1},{"begin":382,"length":1},{"begin":385,"length":4},{"begin":393,"length":1},{"begin":396,"length":1},{"begin":401,"length":1},{"begin":403,"length":1},{"begin":405,"length":1},{"begin":407,"length":1},{"begin":409,"length":1},{"begin":411,"length":2},{"begin":415,"length":3},{"begin":420,"length":1},{"begin":422,"length":2},{"begin":425,"length":4},{"begin":432,"length":1},{"begin":438,"length":4},{"begin":443,"length":2},{"begin":446,"length":1},{"begin":450,"length":1},{"begin":454,"length":1},{"begin":456,"length":2},{"begin":460,"length":2},{"begin":463,"length":3},{"begin":467,"length":1},{"begin":469,"length":2},{"begin":473,"length":1},{"begin":478,"length":1},{"begin":480,"length":1},{"begin":482,"length":8},{"begin":492,"length":4},{"begin":497,"length":1},{"begin":499,"length":2},{"begin":502,"length":2},{"begin":506,"length":1},{"begin":509,"length":1},{"begin":512,"length":3},{"begin":517,"length":1},{"begin":520,"length":3},{"begin":529,"length":4},{"begin":534,"length":1},{"begin":537,"length":1},{"begin":539,"length":3},{"begin":543,"length":8},{"begin":552,"length":1},{"begin":555,"length":2},{"begin":558,"length":2},{"begin":562,"length":2},{"begin":565,"length":2},{"begin":568,"length":1},{"begin":572,"length":3},{"begin":576,"length":4},{"begin":581,"length":1},{"begin":583,"length":1},{"begin":587,"length":1},{"begin":594,"length":1},{"begin":598,"length":1},{"begin":600,"length":5},{"begin":606,"length":1},{"begin":608,"length":1},{"begin":611,"length":1},{"begin":619,"length":2},{"begin":623,"length":3},{"begin":629,"length":2},{"begin":632,"length":1},{"begin":634,"length":1},{"begin":636,"length":2},{"begin":640,"length":2},{"begin":644,"length":1},{"begin":648,"length":3},{"begin":652,"length":1},{"begin":657,"length":1},{"begin":659,"length":1},{"begin":671,"length":1},{"begin":673,"length":3},{"begin":678,"length":1},{"begin":685,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-06-08T07:35:43.185 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon ceph.osd.0 is failed for ~281s , 2023-06-08T07:35:43.329 DEBUG:teuthology.orchestra.run.smithi099:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json Ignoring malparsed backtrace: r = self._runner(client=self.ssh, name=self.shortname, **kwargs) , File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/orchestra/run.py", line 455, in run , r.wait() Ignoring malparsed backtrace: File "/home/teuthworker/src/github.com_ceph_ceph_ea807834105141b6938ce6a9dba28667420efa2e/qa/tasks/ceph.py", line 1469, in run_daemon , yield , File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/contextutil.py", line 32, in nested Ignoring malparsed backtrace: 2023-06-08T08:28:13.377 DEBUG:teuthology.orchestra.run.smithi099:> sync && sudo umount -f /var/lib/ceph/osd/ceph-1 , 2023-06-08T08:28:13.453 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-2 on ubuntu@smithi162.front.sepia.ceph.com , 2023-06-08T08:28:13.453 DEBUG:teuthology.orchestra.run.smithi162:> sync && sudo umount -f /var/lib/ceph/osd/ceph-2 Ignoring malparsed backtrace: /var/log/ceph/ceph-client.admin.127918.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.121150.log , 50.0% -- replaced with /var/log/ceph/ceph-client.admin.127918.log.gz , /var/log/ceph/ceph-client.admin.130308.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.123182.log Ignoring malparsed backtrace: 49.6% -- replaced with /var/log/ceph/ceph-client.admin.104245.log.gz , /var/log/ceph/ceph-client.admin.120435.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.117780.log , 50.0% -- replaced with /var/log/ceph/ceph-client.admin.120435.log.gz Ignoring malparsed backtrace: /var/log/ceph/ceph-client.admin.109652.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.124567.log , 50.0% -- replaced with /var/log/ceph/ceph-client.admin.109652.log.gz , /var/log/ceph/ceph-client.admin.104343.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.126564.log Ignoring malparsed backtrace: 50.0% -- replaced with /var/log/ceph/ceph-client.admin.128995.log.gz , /var/log/ceph/ceph-client.admin.122117.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.112523.log , 50.4% -- replaced with /var/log/ceph/ceph-client.admin.122117.log.gz Ignoring malparsed backtrace: /var/log/ceph/ceph-client.admin.125535.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.127679.log , 50.0% -- replaced with /var/log/ceph/ceph-client.admin.125535.log.gz , /var/log/ceph/ceph-client.admin.132894.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.103886.log Ignoring malparsed backtrace: /var/log/ceph/ceph-client.admin.111570.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.132032.log , 50.4% -- replaced with /var/log/ceph/ceph-client.admin.111570.log.gz , /var/log/ceph/ceph-client.admin.105643.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.135368.log Ignoring malparsed backtrace: 49.6% -- replaced with /var/log/ceph/ceph-client.admin.115634.log.gz , /var/log/ceph/ceph-client.admin.105211.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.110324.log , 50.0% -- replaced with /var/log/ceph/ceph-client.admin.105211.log.gz Ignoring malparsed backtrace: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.114661.log , /var/log/ceph/ceph-client.admin.103574.log: 50.0% -- replaced with /var/log/ceph/ceph-client.admin.103574.log.gz , /var/log/ceph/ceph-client.admin.133490.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.110814.log Ignoring malparsed backtrace: 50.4% -- replaced with /var/log/ceph/ceph-client.admin.112250.log.gz , /var/log/ceph/ceph-client.admin.120716.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.118263.log , 50.0% -- replaced with /var/log/ceph/ceph-client.admin.120716.log.gz Ignoring malparsed backtrace: 50.0% -- replaced with /var/log/ceph/ceph-client.admin.131038.log.gz , /var/log/ceph/ceph-client.admin.126772.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.111997.log , 50.0% -- replaced with /var/log/ceph/ceph-client.admin.126772.log.gz Ignoring malparsed backtrace: /var/log/ceph/ceph-client.admin.111786.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.132208.log , 49.2% -- replaced with /var/log/ceph/ceph-client.admin.111786.log.gz , /var/log/ceph/ceph-client.admin.113152.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.109491.log Ignoring malparsed backtrace: 50.0% -- replaced with /var/log/ceph/ceph-client.admin.114283.log.gz , /var/log/ceph/ceph-client.admin.125913.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.105037.log , 50.4% -- replaced with /var/log/ceph/ceph-client.admin.125913.log.gz Ignoring malparsed backtrace: sys 0m0.444s , 92.7% -- replaced with /var/log/ceph/ceph-osd.3.log.gz , Ignoring malparsed backtrace: 2023-06-08T08:28:21.213 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive , 2023-06-08T08:28:21.227 INFO:teuthology.task.internal:Transferring archived files... , 2023-06-08T08:28:21.229 DEBUG:teuthology.misc:Transferring archived files from smithi099:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/matan-2023-06-08_06:36:16-crimson-rados-wip-matanb-crimson-testing-bug-61504-distro-crimson-smithi/7298180/remote/smithi099 Ignoring malparsed backtrace: ceph version 18.0.0-4330-g9b3f3761 (9b3f3761ef331c3cedab822629c5de78b7d54292) reef (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7f79217aded4] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x1b0) [0x5624be8ab0ca] Ignoring malparsed backtrace: 2023-06-08T07:35:59.680+0000 7f99cf204700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-06-08T07:35:59.689+0000 7f99cdfa2700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , set osd.3 primary-affinity to 1 (8655362) Ignoring malparsed backtrace: 2023-06-08T07:36:57.985+0000 7fc70547d700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-06-08T07:36:57.991+0000 7fc70547d700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-06-08T07:36:58.195 INFO:teuthology.orchestra.run.smithi088.stdout: Ignoring malparsed backtrace: 2023-06-08T07:37:58.478+0000 7f5135e73700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-06-08T07:37:58.484+0000 7f5135e73700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-06-08T07:37:58.690 INFO:teuthology.orchestra.run.smithi088.stdout: Ignoring malparsed backtrace: 2023-06-08T07:38:58.816+0000 7f58d3781700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-06-08T07:38:58.822+0000 7f58d3781700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-06-08T07:38:59.034 INFO:teuthology.orchestra.run.smithi088.stdout: Ignoring malparsed backtrace: 2023-06-08T07:39:59.330+0000 7f457f6ef700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-06-08T07:39:59.337+0000 7f457f6ef700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-06-08T07:39:59.544 INFO:teuthology.orchestra.run.smithi088.stdout: Ignoring malparsed backtrace: File "/home/teuthworker/src/github.com_ceph_ceph_ea807834105141b6938ce6a9dba28667420efa2e/qa/tasks/ceph_manager.py", line 2531, in get_osd_dump , return self.get_osd_dump_json()['osds'] , File "/home/teuthworker/src/github.com_ceph_ceph_ea807834105141b6938ce6a9dba28667420efa2e/qa/tasks/ceph_manager.py", line 2523, in get_osd_dump_json Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_teuthology_2ef0dcdee569e9851c9d2ffb6acfd4b80d031e61/teuthology/orchestra/run.py", line 181, in _raise_for_status , raise CommandFailedError( , teuthology.exceptions.CommandFailedError: Command failed on smithi088 with status 1: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 0 2>> /var/log/ceph/ceph-osd.0.log' Ignoring malparsed backtrace: /var/log/ceph/ceph-client.admin.120345.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.122292.log , 49.6% -- replaced with /var/log/ceph/ceph-client.admin.120345.log.gz , /var/log/ceph/ceph-client.admin.114061.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.117093.log Ignoring malparsed backtrace: /var/log/ceph/ceph-client.admin.105244.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.108828.log , 49.6% -- replaced with /var/log/ceph/ceph-client.admin.105244.log.gz , /var/log/ceph/ceph-client.admin.125585.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.103920.log Ignoring malparsed backtrace: 50.0% -- replaced with /var/log/ceph/ceph-client.admin.107686.log.gz , /var/log/ceph/ceph-client.admin.128408.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.128644.log , 50.0% -- replaced with /var/log/ceph/ceph-client.admin.128408.log.gz Ignoring malparsed backtrace: /var/log/ceph/ceph-client.admin.126383.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.119917.log , 50.0% -- replaced with /var/log/ceph/ceph-client.admin.126383.log.gz , /var/log/ceph/ceph-client.admin.137994.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.128457.log Ignoring malparsed backtrace: 50.0% -- replaced with /var/log/ceph/ceph-client.admin.127013.log.gz , /var/log/ceph/ceph-client.admin.124700.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.143750.log , 50.4% -- replaced with /var/log/ceph/ceph-client.admin.124700.log.gz Ignoring malparsed backtrace: /var/log/ceph/ceph-client.admin.125802.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.104923.log , 49.6% -- replaced with /var/log/ceph/ceph-client.admin.125802.log.gz , /var/log/ceph/ceph-client.admin.113683.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.132872.log Ignoring malparsed backtrace: 50.4% -- replaced with /var/log/ceph/ceph-client.admin.133733.log.gz , /var/log/ceph/ceph-client.admin.105168.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.128302.log , 49.6% -- replaced with /var/log/ceph/ceph-client.admin.105168.log.gz Ignoring malparsed backtrace: 47.6% -- replaced with /var/log/ceph/ceph-client.admin.128205.log.gz , /var/log/ceph/ceph-client.admin.110907.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.121592.log , 50.4% -- replaced with /var/log/ceph/ceph-client.admin.110907.log.gz Ignoring malparsed backtrace: /var/log/ceph/ceph-client.admin.143848.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.134663.log , 50.0% -- replaced with /var/log/ceph/ceph-client.admin.143848.log.gz , /var/log/ceph/ceph-client.admin.128562.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.145687.log Ignoring malparsed backtrace: 50.4% -- replaced with /var/log/ceph/ceph-client.admin.114712.log.gz , /var/log/ceph/ceph-client.admin.117640.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.122187.log , 49.6% -- replaced with /var/log/ceph/ceph-client.admin.117640.log.gz Ignoring malparsed backtrace: 50.0% -- replaced with /var/log/ceph/ceph-client.admin.131955.log.gz , /var/log/ceph/ceph-client.admin.141421.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.140459.log , 50.0% -- replaced with /var/log/ceph/ceph-client.admin.141421.log.gz Ignoring malparsed backtrace: 50.0% -- replaced with /var/log/ceph/ceph-client.admin.130841.log.gz , /var/log/ceph/ceph-client.admin.144544.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.132116.log , 49.6% -- replaced with /var/log/ceph/ceph-client.admin.144544.log.gz Ignoring malparsed backtrace: /var/log/ceph/ceph-client.admin.121648.log: gzip -5 --verbose -- /var/log/ceph/ceph.log , 50.0% -- replaced with /var/log/ceph/ceph-client.admin.121648.log.gz , /var/log/ceph/ceph-client.admin.120969.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.138043.log Ignoring malparsed backtrace: 50.4% -- replaced with /var/log/ceph/ceph-client.admin.120451.log.gz , /var/log/ceph/ceph-client.admin.129264.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.120808.log , 50.0% -- replaced with /var/log/ceph/ceph-client.admin.129264.log.gz Ignoring malparsed backtrace: /var/log/ceph/ceph-client.admin.109143.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.131043.log , 50.0% -- replaced with /var/log/ceph/ceph-client.admin.109143.log.gz , /var/log/ceph/ceph-client.admin.107196.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.124878.log Ignoring malparsed backtrace: /var/log/ceph/ceph-client.admin.116981.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.114600.log , 50.4% -- replaced with /var/log/ceph/ceph-client.admin.116981.log.gz , gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.145537.log Ignoring malparsed backtrace: File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__ , next(self.gen) , File "/home/teuthworker/src/github.com_ceph_ceph_ea807834105141b6938ce6a9dba28667420efa2e/qa/tasks/rados.py", line 286, in task Ignoring malparsed backtrace: 2023-06-08T08:33:05.370 INFO:teuthology.orchestra.run.smithi088.stdout:/usr/bin/ceph_test_rados , 2023-06-08T08:33:05.371 DEBUG:teuthology.orchestra.run:got remote process result: 5 , /usr/bin/which: no - in (/home/ubuntu/.local/bin:/home/ubuntu/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin) Found 5 distinct failure reasons Timeout 3h running clone.client.0/qa/workunits/rbd/crimson/test_crimson_librbd.sh 3 jobs: ['7298145', '7298136', '7298135'] suites intersection: ['clusters/fixed-1', 'crimson-rados/rbd/{centos_latest', 'crimson_qa_overrides', 'deploy/ceph'] suites union: ['clusters/fixed-1', 'crimson-rados/rbd/{centos_latest', 'crimson_qa_overrides', 'deploy/ceph', 'tasks/rbd_api_tests_old_format}', 'tasks/rbd_api_tests}'] Failure: Command crashed: 'CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --localize-reads --max-ops 4000 --objects 50 --max-in-flight 16 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 0 --op read 100 --op write 50 --op delete 50 --op snap_create 50 --op snap_remove 50 --op rollback 0 --op copy_from 0 --op write_excl 50 --pool unique_pool_0' 2 jobs: ['7298180', '7298183'] suites intersection: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{default}', 'centos_latest', 'clusters/{fixed-2}', 'crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'crimson_qa_overrides', 'deploy/ceph', 'objectstore/bluestore', 'thrashers/default', 'thrashosds-health'] suites union: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{default}', 'centos_latest', 'clusters/{fixed-2}', 'crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'crimson_qa_overrides', 'deploy/ceph', 'objectstore/bluestore', 'thrashers/default', 'thrashosds-health', 'workloads/snaps-few-objects-localized}', 'workloads/snaps-few-objects}'] Failure: Command failed (workunit test rbd/crimson/test_crimson_librbd.sh) on smithi119 with status 134: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=ea807834105141b6938ce6a9dba28667420efa2e TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 CRIMSON_COMPAT=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/crimson/test_crimson_librbd.sh' 3 jobs: ['7298143', '7298144', '7298134'] suites intersection: ['clusters/fixed-1', 'crimson-rados/rbd/{centos_latest', 'crimson_qa_overrides', 'deploy/ceph'] suites union: ['clusters/fixed-1', 'crimson-rados/rbd/{centos_latest', 'crimson_qa_overrides', 'deploy/ceph', 'tasks/rbd_api_tests_old_format}', 'tasks/rbd_api_tests}'] Timeout 3h running clone.client.0/qa/workunits/rados/test.sh 3 jobs: ['7298131', '7298132', '7298133'] suites intersection: ['clusters/fixed-2', 'crimson-rados/basic/{centos_latest', 'crimson_qa_overrides', 'deploy/ceph', 'tasks/rados_api_tests}'] suites union: ['clusters/fixed-2', 'crimson-rados/basic/{centos_latest', 'crimson_qa_overrides', 'deploy/ceph', 'tasks/rados_api_tests}'] Timeout 120 running ceph --cluster ceph pg dump --format=json 1 jobs: ['7298148'] suites: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{default}', 'centos_latest', 'clusters/{fixed-2}', 'crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'crimson_qa_overrides', 'deploy/ceph', 'objectstore/bluestore', 'thrashers/default', 'thrashosds-health', 'workloads/radosbench-high-concurrency}']