2023-05-23T06:49:17.575 INFO:root:teuthology version: 0.0.1.dev105+g6fb4f0c 2023-05-23T06:49:17.584 INFO:teuthology.results:seed: '1761' 2023-05-23T06:50:07.763 INFO:root:teuthology version: 0.0.1.dev105+g6fb4f0c 2023-05-23T06:50:07.764 INFO:teuthology.results:Waiting up to 43200 seconds for tests to finish... 2023-05-23T10:30:24.746 INFO:teuthology.results:Tests finished! gathering results... 2023-05-23T10:30:26.434 INFO:scrape:Found 14 jobs 2023-05-23T10:30:27.668 WARNING:scrape:Ignoring malparsed backtrace: ceph version 18.0.0-3565-g031fa640 (031fa64016bfbb21467aeca1fc33541e74a27742) reef (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7fe8aa7d6ed4] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x61b) [0x564c93805553] 2023-05-23T10:30:27.683 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T07:23:14.422+0000 7fb9be536700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-05-23T07:23:14.429+0000 7fb9be536700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-05-23T07:23:14.628 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-05-23T10:30:27.684 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T07:24:08.110+0000 7f4a9f2a5700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-05-23T07:24:08.312 INFO:teuthology.orchestra.run.smithi144.stdout: , 2023-05-23T07:24:08.313 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":29,"fsid":"7d338af8-63f4-4900-b009-053531a81213","created":"2023-05-23T07:22:06.243746+0000","modified":"2023-05-23T07:23:14.276188+0000","last_up_change":"2023-05-23T07:23:02.216077+0000","last_in_change":"2023-05-23T07:22:08.484361+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-05-23T07:22:14.589614+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-05-23T07:22:16.773560+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":3,"score_stable":3,"optimal_score":0.5,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-05-23T07:22:23.573760+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":2.5,"score_stable":2.5,"optimal_score":0.5,"raw_score_acting":1.25,"raw_score_stable":1.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c348b0c2-c869-45f9-83b4-4175565f3837","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":16,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":101503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":101503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":101503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":101503}]},"public_addr":"172.21.15.144:6800/101503","cluster_addr":"172.21.15.144:6801/101503","heartbeat_back_addr":"172.21.15.144:6804/101503","heartbeat_front_addr":"172.21.15.144:6805/101503","state":["exists"]},{"osd":1,"uuid":"d0f0cf63-e126-4142-a372-86e4aed1d7a7","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":16,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":101530}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":101530}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":101530}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":101530}]},"public_addr":"172.21.15.144:6803/101530","cluster_addr":"172.21.15.144:6802/101530","heartbeat_back_addr":"172.21.15.144:6806/101530","heartbeat_front_addr":"172.21.15.144:6807/101530","state":["exists"]},{"osd":2,"uuid":"62a182e0-c5b0-4de3-817a-be3badf82cab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6800","nonce":101122}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6801","nonce":101122}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6808","nonce":101122}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6809","nonce":101122}]},"public_addr":"172.21.15.160:6800/101122","cluster_addr":"172.21.15.160:6801/101122","heartbeat_back_addr":"172.21.15.160:6808/101122","heartbeat_front_addr":"172.21.15.160:6809/101122","state":["exists","up"]},{"osd":3,"uuid":"04b2f63b-9cfa-47d6-89dc-b70ab6c22363","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6802","nonce":101150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6803","nonce":101150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6807","nonce":101150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6806","nonce":101150}]},"public_addr":"172.21.15.160:6802/101150","cluster_addr":"172.21.15.160:6803/101150","heartbeat_back_addr":"172.21.15.160:6807/101150","heartbeat_front_addr":"172.21.15.160:6806/101150","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-05-23T07:23:02.216077+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-05-23T07:23:02.216077+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-05-23T10:30:27.685 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T07:25:01.988 INFO:teuthology.orchestra.run.smithi144.stdout: , 2023-05-23T07:25:01.988 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":29,"fsid":"7d338af8-63f4-4900-b009-053531a81213","created":"2023-05-23T07:22:06.243746+0000","modified":"2023-05-23T07:23:14.276188+0000","last_up_change":"2023-05-23T07:23:02.216077+0000","last_in_change":"2023-05-23T07:22:08.484361+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-05-23T07:22:14.589614+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-05-23T07:22:16.773560+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":3,"score_stable":3,"optimal_score":0.5,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-05-23T07:22:23.573760+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":2.5,"score_stable":2.5,"optimal_score":0.5,"raw_score_acting":1.25,"raw_score_stable":1.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c348b0c2-c869-45f9-83b4-4175565f3837","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":16,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":101503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":101503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":101503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":101503}]},"public_addr":"172.21.15.144:6800/101503","cluster_addr":"172.21.15.144:6801/101503","heartbeat_back_addr":"172.21.15.144:6804/101503","heartbeat_front_addr":"172.21.15.144:6805/101503","state":["exists"]},{"osd":1,"uuid":"d0f0cf63-e126-4142-a372-86e4aed1d7a7","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":16,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":101530}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":101530}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":101530}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":101530}]},"public_addr":"172.21.15.144:6803/101530","cluster_addr":"172.21.15.144:6802/101530","heartbeat_back_addr":"172.21.15.144:6806/101530","heartbeat_front_addr":"172.21.15.144:6807/101530","state":["exists"]},{"osd":2,"uuid":"62a182e0-c5b0-4de3-817a-be3badf82cab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6800","nonce":101122}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6801","nonce":101122}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6808","nonce":101122}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6809","nonce":101122}]},"public_addr":"172.21.15.160:6800/101122","cluster_addr":"172.21.15.160:6801/101122","heartbeat_back_addr":"172.21.15.160:6808/101122","heartbeat_front_addr":"172.21.15.160:6809/101122","state":["exists","up"]},{"osd":3,"uuid":"04b2f63b-9cfa-47d6-89dc-b70ab6c22363","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6802","nonce":101150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6803","nonce":101150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6807","nonce":101150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6806","nonce":101150}]},"public_addr":"172.21.15.160:6802/101150","cluster_addr":"172.21.15.160:6803/101150","heartbeat_back_addr":"172.21.15.160:6807/101150","heartbeat_front_addr":"172.21.15.160:6806/101150","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-05-23T07:23:02.216077+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-05-23T07:23:02.216077+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-05-23T07:25:05.003 DEBUG:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2023-05-23T10:30:27.686 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T07:25:55.667 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":29,"fsid":"7d338af8-63f4-4900-b009-053531a81213","created":"2023-05-23T07:22:06.243746+0000","modified":"2023-05-23T07:23:14.276188+0000","last_up_change":"2023-05-23T07:23:02.216077+0000","last_in_change":"2023-05-23T07:22:08.484361+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-05-23T07:22:14.589614+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-05-23T07:22:16.773560+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":3,"score_stable":3,"optimal_score":0.5,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-05-23T07:22:23.573760+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":2.5,"score_stable":2.5,"optimal_score":0.5,"raw_score_acting":1.25,"raw_score_stable":1.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c348b0c2-c869-45f9-83b4-4175565f3837","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":16,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":101503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":101503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":101503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":101503}]},"public_addr":"172.21.15.144:6800/101503","cluster_addr":"172.21.15.144:6801/101503","heartbeat_back_addr":"172.21.15.144:6804/101503","heartbeat_front_addr":"172.21.15.144:6805/101503","state":["exists"]},{"osd":1,"uuid":"d0f0cf63-e126-4142-a372-86e4aed1d7a7","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":16,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":101530}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":101530}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":101530}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":101530}]},"public_addr":"172.21.15.144:6803/101530","cluster_addr":"172.21.15.144:6802/101530","heartbeat_back_addr":"172.21.15.144:6806/101530","heartbeat_front_addr":"172.21.15.144:6807/101530","state":["exists"]},{"osd":2,"uuid":"62a182e0-c5b0-4de3-817a-be3badf82cab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6800","nonce":101122}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6801","nonce":101122}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6808","nonce":101122}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6809","nonce":101122}]},"public_addr":"172.21.15.160:6800/101122","cluster_addr":"172.21.15.160:6801/101122","heartbeat_back_addr":"172.21.15.160:6808/101122","heartbeat_front_addr":"172.21.15.160:6809/101122","state":["exists","up"]},{"osd":3,"uuid":"04b2f63b-9cfa-47d6-89dc-b70ab6c22363","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6802","nonce":101150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6803","nonce":101150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6807","nonce":101150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6806","nonce":101150}]},"public_addr":"172.21.15.160:6802/101150","cluster_addr":"172.21.15.160:6803/101150","heartbeat_back_addr":"172.21.15.160:6807/101150","heartbeat_front_addr":"172.21.15.160:6806/101150","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-05-23T07:23:02.216077+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-05-23T07:23:02.216077+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-05-23T07:25:58.681 DEBUG:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-05-23T07:25:58.813+0000 7f4bd8039700 -1 WARNING: the following dangerous and experimental features are enabled: crimson 2023-05-23T10:30:27.687 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T07:26:52.376 DEBUG:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-05-23T07:26:52.510+0000 7f73f343a700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-05-23T07:26:52.517+0000 7f73f343a700 -1 WARNING: the following dangerous and experimental features are enabled: crimson 2023-05-23T10:30:27.688 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T07:27:44.100 INFO:tasks.ceph.osd.2:Sent signal 15 , 2023-05-23T07:27:44.101 INFO:tasks.ceph.osd.3:Sent signal 15 , 2023-05-23T07:27:44.102 INFO:tasks.ceph.mon.a:Sent signal 15 2023-05-23T10:30:27.690 WARNING:scrape:Ignoring malparsed backtrace: proc.wait() , File "/home/teuthworker/src/git.ceph.com_teuthology_6fb4f0c910de3902d27909a642e8d50d731dd28c/teuthology/orchestra/run.py", line 161, in wait , self._raise_for_status() 2023-05-23T10:30:27.691 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T08:19:52.373 DEBUG:tasks.ceph.osd.3:waiting for process to exit , 2023-05-23T08:19:52.373 INFO:teuthology.orchestra.run:waiting for 300 , 2023-05-23T08:19:52.464 INFO:tasks.ceph.osd.3:Stopped 2023-05-23T10:30:27.691 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T08:19:55.424 DEBUG:teuthology.orchestra.run.smithi144:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log , 2023-05-23T08:19:55.525 DEBUG:teuthology.orchestra.run.smithi160:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log , 2023-05-23T08:19:55.612 DEBUG:teuthology.orchestra.run.smithi144:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-05-23T10:30:27.692 WARNING:scrape:Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_teuthology_6fb4f0c910de3902d27909a642e8d50d731dd28c/teuthology/orchestra/run.py", line 161, in wait , self._raise_for_status() , File "/home/teuthworker/src/git.ceph.com_teuthology_6fb4f0c910de3902d27909a642e8d50d731dd28c/teuthology/orchestra/run.py", line 181, in _raise_for_status 2023-05-23T10:30:27.727 WARNING:scrape:Ignoring malparsed backtrace: ceph version 18.0.0-3565-g031fa640 (031fa64016bfbb21467aeca1fc33541e74a27742) reef (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7fe8aa7d6ed4] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x61b) [0x564c93805553] 2023-05-23T10:30:27.730 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T07:23:14.422+0000 7fb9be536700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-05-23T07:23:14.429+0000 7fb9be536700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-05-23T07:23:14.628 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-05-23T10:30:27.731 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T07:24:08.110+0000 7f4a9f2a5700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-05-23T07:24:08.312 INFO:teuthology.orchestra.run.smithi144.stdout: , 2023-05-23T07:24:08.313 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":29,"fsid":"7d338af8-63f4-4900-b009-053531a81213","created":"2023-05-23T07:22:06.243746+0000","modified":"2023-05-23T07:23:14.276188+0000","last_up_change":"2023-05-23T07:23:02.216077+0000","last_in_change":"2023-05-23T07:22:08.484361+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-05-23T07:22:14.589614+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-05-23T07:22:16.773560+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":3,"score_stable":3,"optimal_score":0.5,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-05-23T07:22:23.573760+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":2.5,"score_stable":2.5,"optimal_score":0.5,"raw_score_acting":1.25,"raw_score_stable":1.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c348b0c2-c869-45f9-83b4-4175565f3837","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":16,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":101503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":101503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":101503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":101503}]},"public_addr":"172.21.15.144:6800/101503","cluster_addr":"172.21.15.144:6801/101503","heartbeat_back_addr":"172.21.15.144:6804/101503","heartbeat_front_addr":"172.21.15.144:6805/101503","state":["exists"]},{"osd":1,"uuid":"d0f0cf63-e126-4142-a372-86e4aed1d7a7","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":16,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":101530}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":101530}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":101530}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":101530}]},"public_addr":"172.21.15.144:6803/101530","cluster_addr":"172.21.15.144:6802/101530","heartbeat_back_addr":"172.21.15.144:6806/101530","heartbeat_front_addr":"172.21.15.144:6807/101530","state":["exists"]},{"osd":2,"uuid":"62a182e0-c5b0-4de3-817a-be3badf82cab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6800","nonce":101122}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6801","nonce":101122}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6808","nonce":101122}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6809","nonce":101122}]},"public_addr":"172.21.15.160:6800/101122","cluster_addr":"172.21.15.160:6801/101122","heartbeat_back_addr":"172.21.15.160:6808/101122","heartbeat_front_addr":"172.21.15.160:6809/101122","state":["exists","up"]},{"osd":3,"uuid":"04b2f63b-9cfa-47d6-89dc-b70ab6c22363","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6802","nonce":101150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6803","nonce":101150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6807","nonce":101150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6806","nonce":101150}]},"public_addr":"172.21.15.160:6802/101150","cluster_addr":"172.21.15.160:6803/101150","heartbeat_back_addr":"172.21.15.160:6807/101150","heartbeat_front_addr":"172.21.15.160:6806/101150","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-05-23T07:23:02.216077+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-05-23T07:23:02.216077+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-05-23T10:30:27.733 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T07:25:01.988 INFO:teuthology.orchestra.run.smithi144.stdout: , 2023-05-23T07:25:01.988 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":29,"fsid":"7d338af8-63f4-4900-b009-053531a81213","created":"2023-05-23T07:22:06.243746+0000","modified":"2023-05-23T07:23:14.276188+0000","last_up_change":"2023-05-23T07:23:02.216077+0000","last_in_change":"2023-05-23T07:22:08.484361+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-05-23T07:22:14.589614+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-05-23T07:22:16.773560+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":3,"score_stable":3,"optimal_score":0.5,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-05-23T07:22:23.573760+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":2.5,"score_stable":2.5,"optimal_score":0.5,"raw_score_acting":1.25,"raw_score_stable":1.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c348b0c2-c869-45f9-83b4-4175565f3837","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":16,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":101503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":101503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":101503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":101503}]},"public_addr":"172.21.15.144:6800/101503","cluster_addr":"172.21.15.144:6801/101503","heartbeat_back_addr":"172.21.15.144:6804/101503","heartbeat_front_addr":"172.21.15.144:6805/101503","state":["exists"]},{"osd":1,"uuid":"d0f0cf63-e126-4142-a372-86e4aed1d7a7","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":16,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":101530}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":101530}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":101530}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":101530}]},"public_addr":"172.21.15.144:6803/101530","cluster_addr":"172.21.15.144:6802/101530","heartbeat_back_addr":"172.21.15.144:6806/101530","heartbeat_front_addr":"172.21.15.144:6807/101530","state":["exists"]},{"osd":2,"uuid":"62a182e0-c5b0-4de3-817a-be3badf82cab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6800","nonce":101122}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6801","nonce":101122}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6808","nonce":101122}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6809","nonce":101122}]},"public_addr":"172.21.15.160:6800/101122","cluster_addr":"172.21.15.160:6801/101122","heartbeat_back_addr":"172.21.15.160:6808/101122","heartbeat_front_addr":"172.21.15.160:6809/101122","state":["exists","up"]},{"osd":3,"uuid":"04b2f63b-9cfa-47d6-89dc-b70ab6c22363","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6802","nonce":101150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6803","nonce":101150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6807","nonce":101150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6806","nonce":101150}]},"public_addr":"172.21.15.160:6802/101150","cluster_addr":"172.21.15.160:6803/101150","heartbeat_back_addr":"172.21.15.160:6807/101150","heartbeat_front_addr":"172.21.15.160:6806/101150","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-05-23T07:23:02.216077+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-05-23T07:23:02.216077+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-05-23T07:25:05.003 DEBUG:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2023-05-23T10:30:27.739 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T07:25:55.667 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":29,"fsid":"7d338af8-63f4-4900-b009-053531a81213","created":"2023-05-23T07:22:06.243746+0000","modified":"2023-05-23T07:23:14.276188+0000","last_up_change":"2023-05-23T07:23:02.216077+0000","last_in_change":"2023-05-23T07:22:08.484361+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-05-23T07:22:14.589614+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-05-23T07:22:16.773560+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":3,"score_stable":3,"optimal_score":0.5,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-05-23T07:22:23.573760+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":2.5,"score_stable":2.5,"optimal_score":0.5,"raw_score_acting":1.25,"raw_score_stable":1.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c348b0c2-c869-45f9-83b4-4175565f3837","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":16,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":101503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":101503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":101503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":101503}]},"public_addr":"172.21.15.144:6800/101503","cluster_addr":"172.21.15.144:6801/101503","heartbeat_back_addr":"172.21.15.144:6804/101503","heartbeat_front_addr":"172.21.15.144:6805/101503","state":["exists"]},{"osd":1,"uuid":"d0f0cf63-e126-4142-a372-86e4aed1d7a7","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":16,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":101530}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":101530}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":101530}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":101530}]},"public_addr":"172.21.15.144:6803/101530","cluster_addr":"172.21.15.144:6802/101530","heartbeat_back_addr":"172.21.15.144:6806/101530","heartbeat_front_addr":"172.21.15.144:6807/101530","state":["exists"]},{"osd":2,"uuid":"62a182e0-c5b0-4de3-817a-be3badf82cab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6800","nonce":101122}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6801","nonce":101122}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6808","nonce":101122}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6809","nonce":101122}]},"public_addr":"172.21.15.160:6800/101122","cluster_addr":"172.21.15.160:6801/101122","heartbeat_back_addr":"172.21.15.160:6808/101122","heartbeat_front_addr":"172.21.15.160:6809/101122","state":["exists","up"]},{"osd":3,"uuid":"04b2f63b-9cfa-47d6-89dc-b70ab6c22363","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6802","nonce":101150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6803","nonce":101150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6807","nonce":101150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6806","nonce":101150}]},"public_addr":"172.21.15.160:6802/101150","cluster_addr":"172.21.15.160:6803/101150","heartbeat_back_addr":"172.21.15.160:6807/101150","heartbeat_front_addr":"172.21.15.160:6806/101150","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-05-23T07:23:02.216077+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-05-23T07:23:02.216077+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-05-23T07:25:58.681 DEBUG:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-05-23T07:25:58.813+0000 7f4bd8039700 -1 WARNING: the following dangerous and experimental features are enabled: crimson 2023-05-23T10:30:27.740 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T07:26:52.376 DEBUG:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-05-23T07:26:52.510+0000 7f73f343a700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-05-23T07:26:52.517+0000 7f73f343a700 -1 WARNING: the following dangerous and experimental features are enabled: crimson 2023-05-23T10:30:27.741 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T07:27:44.100 INFO:tasks.ceph.osd.2:Sent signal 15 , 2023-05-23T07:27:44.101 INFO:tasks.ceph.osd.3:Sent signal 15 , 2023-05-23T07:27:44.102 INFO:tasks.ceph.mon.a:Sent signal 15 2023-05-23T10:30:27.741 WARNING:scrape:Ignoring malparsed backtrace: proc.wait() , File "/home/teuthworker/src/git.ceph.com_teuthology_6fb4f0c910de3902d27909a642e8d50d731dd28c/teuthology/orchestra/run.py", line 161, in wait , self._raise_for_status() 2023-05-23T10:30:27.741 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T08:19:52.373 DEBUG:tasks.ceph.osd.3:waiting for process to exit , 2023-05-23T08:19:52.373 INFO:teuthology.orchestra.run:waiting for 300 , 2023-05-23T08:19:52.464 INFO:tasks.ceph.osd.3:Stopped 2023-05-23T10:30:27.742 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T08:19:55.424 DEBUG:teuthology.orchestra.run.smithi144:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log , 2023-05-23T08:19:55.525 DEBUG:teuthology.orchestra.run.smithi160:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log , 2023-05-23T08:19:55.612 DEBUG:teuthology.orchestra.run.smithi144:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-05-23T10:30:27.743 WARNING:scrape:Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_teuthology_6fb4f0c910de3902d27909a642e8d50d731dd28c/teuthology/orchestra/run.py", line 161, in wait , self._raise_for_status() , File "/home/teuthworker/src/git.ceph.com_teuthology_6fb4f0c910de3902d27909a642e8d50d731dd28c/teuthology/orchestra/run.py", line 181, in _raise_for_status 2023-05-23T10:30:27.781 WARNING:scrape:Ignoring malparsed backtrace: ceph version 18.0.0-3565-g031fa640 (031fa64016bfbb21467aeca1fc33541e74a27742) reef (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7fe8aa7d6ed4] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x61b) [0x564c93805553] 2023-05-23T10:30:27.787 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T07:23:14.422+0000 7fb9be536700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-05-23T07:23:14.429+0000 7fb9be536700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-05-23T07:23:14.628 INFO:teuthology.orchestra.run.smithi144.stdout: 2023-05-23T10:30:27.789 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T07:24:08.110+0000 7f4a9f2a5700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-05-23T07:24:08.312 INFO:teuthology.orchestra.run.smithi144.stdout: , 2023-05-23T07:24:08.313 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":29,"fsid":"7d338af8-63f4-4900-b009-053531a81213","created":"2023-05-23T07:22:06.243746+0000","modified":"2023-05-23T07:23:14.276188+0000","last_up_change":"2023-05-23T07:23:02.216077+0000","last_in_change":"2023-05-23T07:22:08.484361+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-05-23T07:22:14.589614+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-05-23T07:22:16.773560+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":3,"score_stable":3,"optimal_score":0.5,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-05-23T07:22:23.573760+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":2.5,"score_stable":2.5,"optimal_score":0.5,"raw_score_acting":1.25,"raw_score_stable":1.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c348b0c2-c869-45f9-83b4-4175565f3837","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":16,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":101503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":101503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":101503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":101503}]},"public_addr":"172.21.15.144:6800/101503","cluster_addr":"172.21.15.144:6801/101503","heartbeat_back_addr":"172.21.15.144:6804/101503","heartbeat_front_addr":"172.21.15.144:6805/101503","state":["exists"]},{"osd":1,"uuid":"d0f0cf63-e126-4142-a372-86e4aed1d7a7","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":16,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":101530}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":101530}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":101530}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":101530}]},"public_addr":"172.21.15.144:6803/101530","cluster_addr":"172.21.15.144:6802/101530","heartbeat_back_addr":"172.21.15.144:6806/101530","heartbeat_front_addr":"172.21.15.144:6807/101530","state":["exists"]},{"osd":2,"uuid":"62a182e0-c5b0-4de3-817a-be3badf82cab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6800","nonce":101122}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6801","nonce":101122}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6808","nonce":101122}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6809","nonce":101122}]},"public_addr":"172.21.15.160:6800/101122","cluster_addr":"172.21.15.160:6801/101122","heartbeat_back_addr":"172.21.15.160:6808/101122","heartbeat_front_addr":"172.21.15.160:6809/101122","state":["exists","up"]},{"osd":3,"uuid":"04b2f63b-9cfa-47d6-89dc-b70ab6c22363","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6802","nonce":101150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6803","nonce":101150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6807","nonce":101150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6806","nonce":101150}]},"public_addr":"172.21.15.160:6802/101150","cluster_addr":"172.21.15.160:6803/101150","heartbeat_back_addr":"172.21.15.160:6807/101150","heartbeat_front_addr":"172.21.15.160:6806/101150","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-05-23T07:23:02.216077+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-05-23T07:23:02.216077+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-05-23T10:30:27.790 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T07:25:01.988 INFO:teuthology.orchestra.run.smithi144.stdout: , 2023-05-23T07:25:01.988 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":29,"fsid":"7d338af8-63f4-4900-b009-053531a81213","created":"2023-05-23T07:22:06.243746+0000","modified":"2023-05-23T07:23:14.276188+0000","last_up_change":"2023-05-23T07:23:02.216077+0000","last_in_change":"2023-05-23T07:22:08.484361+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-05-23T07:22:14.589614+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-05-23T07:22:16.773560+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":3,"score_stable":3,"optimal_score":0.5,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-05-23T07:22:23.573760+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":2.5,"score_stable":2.5,"optimal_score":0.5,"raw_score_acting":1.25,"raw_score_stable":1.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c348b0c2-c869-45f9-83b4-4175565f3837","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":16,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":101503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":101503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":101503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":101503}]},"public_addr":"172.21.15.144:6800/101503","cluster_addr":"172.21.15.144:6801/101503","heartbeat_back_addr":"172.21.15.144:6804/101503","heartbeat_front_addr":"172.21.15.144:6805/101503","state":["exists"]},{"osd":1,"uuid":"d0f0cf63-e126-4142-a372-86e4aed1d7a7","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":16,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":101530}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":101530}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":101530}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":101530}]},"public_addr":"172.21.15.144:6803/101530","cluster_addr":"172.21.15.144:6802/101530","heartbeat_back_addr":"172.21.15.144:6806/101530","heartbeat_front_addr":"172.21.15.144:6807/101530","state":["exists"]},{"osd":2,"uuid":"62a182e0-c5b0-4de3-817a-be3badf82cab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6800","nonce":101122}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6801","nonce":101122}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6808","nonce":101122}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6809","nonce":101122}]},"public_addr":"172.21.15.160:6800/101122","cluster_addr":"172.21.15.160:6801/101122","heartbeat_back_addr":"172.21.15.160:6808/101122","heartbeat_front_addr":"172.21.15.160:6809/101122","state":["exists","up"]},{"osd":3,"uuid":"04b2f63b-9cfa-47d6-89dc-b70ab6c22363","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6802","nonce":101150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6803","nonce":101150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6807","nonce":101150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6806","nonce":101150}]},"public_addr":"172.21.15.160:6802/101150","cluster_addr":"172.21.15.160:6803/101150","heartbeat_back_addr":"172.21.15.160:6807/101150","heartbeat_front_addr":"172.21.15.160:6806/101150","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-05-23T07:23:02.216077+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-05-23T07:23:02.216077+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-05-23T07:25:05.003 DEBUG:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2023-05-23T10:30:27.792 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T07:25:55.667 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":29,"fsid":"7d338af8-63f4-4900-b009-053531a81213","created":"2023-05-23T07:22:06.243746+0000","modified":"2023-05-23T07:23:14.276188+0000","last_up_change":"2023-05-23T07:23:02.216077+0000","last_in_change":"2023-05-23T07:22:08.484361+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-05-23T07:22:14.589614+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-05-23T07:22:16.773560+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":3,"score_stable":3,"optimal_score":0.5,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-05-23T07:22:23.573760+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":2.5,"score_stable":2.5,"optimal_score":0.5,"raw_score_acting":1.25,"raw_score_stable":1.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c348b0c2-c869-45f9-83b4-4175565f3837","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":16,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":101503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6801","nonce":101503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":101503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6805","nonce":101503}]},"public_addr":"172.21.15.144:6800/101503","cluster_addr":"172.21.15.144:6801/101503","heartbeat_back_addr":"172.21.15.144:6804/101503","heartbeat_front_addr":"172.21.15.144:6805/101503","state":["exists"]},{"osd":1,"uuid":"d0f0cf63-e126-4142-a372-86e4aed1d7a7","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":16,"down_at":20,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6803","nonce":101530}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":101530}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":101530}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6807","nonce":101530}]},"public_addr":"172.21.15.144:6803/101530","cluster_addr":"172.21.15.144:6802/101530","heartbeat_back_addr":"172.21.15.144:6806/101530","heartbeat_front_addr":"172.21.15.144:6807/101530","state":["exists"]},{"osd":2,"uuid":"62a182e0-c5b0-4de3-817a-be3badf82cab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6800","nonce":101122}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6801","nonce":101122}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6808","nonce":101122}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6809","nonce":101122}]},"public_addr":"172.21.15.160:6800/101122","cluster_addr":"172.21.15.160:6801/101122","heartbeat_back_addr":"172.21.15.160:6808/101122","heartbeat_front_addr":"172.21.15.160:6809/101122","state":["exists","up"]},{"osd":3,"uuid":"04b2f63b-9cfa-47d6-89dc-b70ab6c22363","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":20,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6802","nonce":101150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6803","nonce":101150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6807","nonce":101150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.160:6806","nonce":101150}]},"public_addr":"172.21.15.160:6802/101150","cluster_addr":"172.21.15.160:6803/101150","heartbeat_back_addr":"172.21.15.160:6807/101150","heartbeat_front_addr":"172.21.15.160:6806/101150","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2023-05-23T07:23:02.216077+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2023-05-23T07:23:02.216077+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-05-23T07:25:58.681 DEBUG:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-05-23T07:25:58.813+0000 7f4bd8039700 -1 WARNING: the following dangerous and experimental features are enabled: crimson 2023-05-23T10:30:27.793 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T07:26:52.376 DEBUG:teuthology.orchestra.run.smithi144:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2023-05-23T07:26:52.510+0000 7f73f343a700 -1 WARNING: the following dangerous and experimental features are enabled: crimson , 2023-05-23T07:26:52.517+0000 7f73f343a700 -1 WARNING: the following dangerous and experimental features are enabled: crimson 2023-05-23T10:30:27.794 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T07:27:44.100 INFO:tasks.ceph.osd.2:Sent signal 15 , 2023-05-23T07:27:44.101 INFO:tasks.ceph.osd.3:Sent signal 15 , 2023-05-23T07:27:44.102 INFO:tasks.ceph.mon.a:Sent signal 15 2023-05-23T10:30:27.794 WARNING:scrape:Ignoring malparsed backtrace: proc.wait() , File "/home/teuthworker/src/git.ceph.com_teuthology_6fb4f0c910de3902d27909a642e8d50d731dd28c/teuthology/orchestra/run.py", line 161, in wait , self._raise_for_status() 2023-05-23T10:30:27.795 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T08:19:52.373 DEBUG:tasks.ceph.osd.3:waiting for process to exit , 2023-05-23T08:19:52.373 INFO:teuthology.orchestra.run:waiting for 300 , 2023-05-23T08:19:52.464 INFO:tasks.ceph.osd.3:Stopped 2023-05-23T10:30:27.795 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T08:19:55.424 DEBUG:teuthology.orchestra.run.smithi144:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log , 2023-05-23T08:19:55.525 DEBUG:teuthology.orchestra.run.smithi160:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log , 2023-05-23T08:19:55.612 DEBUG:teuthology.orchestra.run.smithi144:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-05-23T10:30:27.815 WARNING:scrape:Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_teuthology_6fb4f0c910de3902d27909a642e8d50d731dd28c/teuthology/orchestra/run.py", line 161, in wait , self._raise_for_status() , File "/home/teuthworker/src/git.ceph.com_teuthology_6fb4f0c910de3902d27909a642e8d50d731dd28c/teuthology/orchestra/run.py", line 181, in _raise_for_status 2023-05-23T10:30:30.501 WARNING:scrape:Ignoring malparsed backtrace: ceph version 18.0.0-3565-g031fa640 (031fa64016bfbb21467aeca1fc33541e74a27742) reef (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7f6f7df61ed4] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x1b0) [0x55bda59ae0e8] 2023-05-23T10:30:30.502 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T07:24:46.450 INFO:teuthology.orchestra.run.smithi047.stdout: "success": "" , 2023-05-23T07:24:46.450 INFO:teuthology.orchestra.run.smithi047.stdout:} , 2023-05-23T07:24:46.462 DEBUG:teuthology.orchestra.run.smithi047:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 2 2023-05-23T10:30:30.503 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T07:25:20.830 INFO:tasks.ceph.osd.3:Stopped , 2023-05-23T07:25:20.830 INFO:teuthology.misc:Shutting down mgr daemons... , 2023-05-23T07:25:20.831 DEBUG:tasks.ceph.mgr.x:waiting for process to exit 2023-05-23T10:30:30.503 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T07:25:22.577 DEBUG:teuthology.orchestra.run.smithi047:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 , 2023-05-23T07:25:22.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 , 2023-05-23T07:25:22.610 DEBUG:teuthology.orchestra.run.smithi158:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-05-23T10:30:30.504 WARNING:scrape:Ignoring malparsed backtrace: r.wait() , File "/home/teuthworker/src/git.ceph.com_teuthology_6fb4f0c910de3902d27909a642e8d50d731dd28c/teuthology/orchestra/run.py", line 161, in wait , self._raise_for_status() 2023-05-23T10:30:30.534 WARNING:scrape:Ignoring malparsed backtrace: ceph version 18.0.0-3565-g031fa640 (031fa64016bfbb21467aeca1fc33541e74a27742) reef (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7f6f7df61ed4] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x1b0) [0x55bda59ae0e8] 2023-05-23T10:30:30.536 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T07:24:46.450 INFO:teuthology.orchestra.run.smithi047.stdout: "success": "" , 2023-05-23T07:24:46.450 INFO:teuthology.orchestra.run.smithi047.stdout:} , 2023-05-23T07:24:46.462 DEBUG:teuthology.orchestra.run.smithi047:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 2 2023-05-23T10:30:30.558 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T07:25:20.830 INFO:tasks.ceph.osd.3:Stopped , 2023-05-23T07:25:20.830 INFO:teuthology.misc:Shutting down mgr daemons... , 2023-05-23T07:25:20.831 DEBUG:tasks.ceph.mgr.x:waiting for process to exit 2023-05-23T10:30:30.559 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T07:25:22.577 DEBUG:teuthology.orchestra.run.smithi047:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 , 2023-05-23T07:25:22.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 , 2023-05-23T07:25:22.610 DEBUG:teuthology.orchestra.run.smithi158:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-05-23T10:30:30.561 WARNING:scrape:Ignoring malparsed backtrace: r.wait() , File "/home/teuthworker/src/git.ceph.com_teuthology_6fb4f0c910de3902d27909a642e8d50d731dd28c/teuthology/orchestra/run.py", line 161, in wait , self._raise_for_status() 2023-05-23T10:30:30.576 WARNING:scrape:Ignoring malparsed backtrace: ceph version 18.0.0-3565-g031fa640 (031fa64016bfbb21467aeca1fc33541e74a27742) reef (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7f6f7df61ed4] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x1b0) [0x55bda59ae0e8] 2023-05-23T10:30:30.578 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T07:24:46.450 INFO:teuthology.orchestra.run.smithi047.stdout: "success": "" , 2023-05-23T07:24:46.450 INFO:teuthology.orchestra.run.smithi047.stdout:} , 2023-05-23T07:24:46.462 DEBUG:teuthology.orchestra.run.smithi047:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 2 2023-05-23T10:30:30.578 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T07:25:20.830 INFO:tasks.ceph.osd.3:Stopped , 2023-05-23T07:25:20.830 INFO:teuthology.misc:Shutting down mgr daemons... , 2023-05-23T07:25:20.831 DEBUG:tasks.ceph.mgr.x:waiting for process to exit 2023-05-23T10:30:30.579 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T07:25:22.577 DEBUG:teuthology.orchestra.run.smithi047:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 , 2023-05-23T07:25:22.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 , 2023-05-23T07:25:22.610 DEBUG:teuthology.orchestra.run.smithi158:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-05-23T10:30:30.580 WARNING:scrape:Ignoring malparsed backtrace: r.wait() , File "/home/teuthworker/src/git.ceph.com_teuthology_6fb4f0c910de3902d27909a642e8d50d731dd28c/teuthology/orchestra/run.py", line 161, in wait , self._raise_for_status() 2023-05-23T10:30:30.653 WARNING:scrape:Ignoring malparsed backtrace: ceph version 18.0.0-3565-g031fa640 (031fa64016bfbb21467aeca1fc33541e74a27742) reef (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7f6f7df61ed4] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x1b0) [0x55bda59ae0e8] 2023-05-23T10:30:30.655 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T07:24:46.450 INFO:teuthology.orchestra.run.smithi047.stdout: "success": "" , 2023-05-23T07:24:46.450 INFO:teuthology.orchestra.run.smithi047.stdout:} , 2023-05-23T07:24:46.462 DEBUG:teuthology.orchestra.run.smithi047:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 2 2023-05-23T10:30:30.656 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T07:25:20.830 INFO:tasks.ceph.osd.3:Stopped , 2023-05-23T07:25:20.830 INFO:teuthology.misc:Shutting down mgr daemons... , 2023-05-23T07:25:20.831 DEBUG:tasks.ceph.mgr.x:waiting for process to exit 2023-05-23T10:30:30.666 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T07:25:22.577 DEBUG:teuthology.orchestra.run.smithi047:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 , 2023-05-23T07:25:22.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 , 2023-05-23T07:25:22.610 DEBUG:teuthology.orchestra.run.smithi158:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-05-23T10:30:30.667 WARNING:scrape:Ignoring malparsed backtrace: r.wait() , File "/home/teuthworker/src/git.ceph.com_teuthology_6fb4f0c910de3902d27909a642e8d50d731dd28c/teuthology/orchestra/run.py", line 161, in wait , self._raise_for_status() 2023-05-23T10:30:30.728 WARNING:scrape:Ignoring malparsed backtrace: ceph version 18.0.0-3565-g031fa640 (031fa64016bfbb21467aeca1fc33541e74a27742) reef (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7f6f7df61ed4] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x1b0) [0x55bda59ae0e8] 2023-05-23T10:30:30.736 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T07:24:46.450 INFO:teuthology.orchestra.run.smithi047.stdout: "success": "" , 2023-05-23T07:24:46.450 INFO:teuthology.orchestra.run.smithi047.stdout:} , 2023-05-23T07:24:46.462 DEBUG:teuthology.orchestra.run.smithi047:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 2 2023-05-23T10:30:30.736 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T07:25:20.830 INFO:tasks.ceph.osd.3:Stopped , 2023-05-23T07:25:20.830 INFO:teuthology.misc:Shutting down mgr daemons... , 2023-05-23T07:25:20.831 DEBUG:tasks.ceph.mgr.x:waiting for process to exit 2023-05-23T10:30:30.737 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T07:25:22.577 DEBUG:teuthology.orchestra.run.smithi047:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 , 2023-05-23T07:25:22.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 , 2023-05-23T07:25:22.610 DEBUG:teuthology.orchestra.run.smithi158:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-05-23T10:30:30.738 WARNING:scrape:Ignoring malparsed backtrace: r.wait() , File "/home/teuthworker/src/git.ceph.com_teuthology_6fb4f0c910de3902d27909a642e8d50d731dd28c/teuthology/orchestra/run.py", line 161, in wait , self._raise_for_status() 2023-05-23T10:30:30.817 WARNING:scrape:Ignoring malparsed backtrace: ceph version 18.0.0-3565-g031fa640 (031fa64016bfbb21467aeca1fc33541e74a27742) reef (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7f6f7df61ed4] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x1b0) [0x55bda59ae0e8] 2023-05-23T10:30:30.819 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T07:24:46.450 INFO:teuthology.orchestra.run.smithi047.stdout: "success": "" , 2023-05-23T07:24:46.450 INFO:teuthology.orchestra.run.smithi047.stdout:} , 2023-05-23T07:24:46.462 DEBUG:teuthology.orchestra.run.smithi047:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 2 2023-05-23T10:30:30.819 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T07:25:20.830 INFO:tasks.ceph.osd.3:Stopped , 2023-05-23T07:25:20.830 INFO:teuthology.misc:Shutting down mgr daemons... , 2023-05-23T07:25:20.831 DEBUG:tasks.ceph.mgr.x:waiting for process to exit 2023-05-23T10:30:30.820 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T07:25:22.577 DEBUG:teuthology.orchestra.run.smithi047:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 , 2023-05-23T07:25:22.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 , 2023-05-23T07:25:22.610 DEBUG:teuthology.orchestra.run.smithi158:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-05-23T10:30:30.820 WARNING:scrape:Ignoring malparsed backtrace: r.wait() , File "/home/teuthworker/src/git.ceph.com_teuthology_6fb4f0c910de3902d27909a642e8d50d731dd28c/teuthology/orchestra/run.py", line 161, in wait , self._raise_for_status() 2023-05-23T10:30:30.959 WARNING:scrape:Ignoring malparsed backtrace: ceph version 18.0.0-3565-g031fa640 (031fa64016bfbb21467aeca1fc33541e74a27742) reef (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7f15a4f66ed4] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x1b0) [0x5557514ac0e8] 2023-05-23T10:30:30.967 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T07:21:05.550 INFO:teuthology.orchestra.run.smithi016.stdout: , 2023-05-23T07:21:05.550 INFO:teuthology.orchestra.run.smithi016.stdout:{"epoch":27,"fsid":"9f022d32-3da5-4eee-9db2-a28158115119","created":"2023-05-23T07:20:33.820396+0000","modified":"2023-05-23T07:21:04.826950+0000","last_up_change":"2023-05-23T07:20:41.983751+0000","last_in_change":"2023-05-23T07:20:36.102762+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":true,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-05-23T07:20:42.273797+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.75,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-05-23T07:20:44.422943+0000","flags":262177,"flags_names":"hashpspool,nopgchange,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2023-05-23T07:20:51.600981+0000","flags":262305,"flags_names":"hashpspool,nopgchange,write_fadvise_dontneed,crimson","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"19","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_acting":1,"score_stable":1,"optimal_score":1,"raw_score_acting":1,"raw_score_stable":1,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"a91ffc0d-8fa3-4369-9749-cddd68765d10","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":16,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6800","nonce":101543}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6801","nonce":101543}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6804","nonce":101543}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6805","nonce":101543}]},"public_addr":"172.21.15.16:6800/101543","cluster_addr":"172.21.15.16:6801/101543","heartbeat_back_addr":"172.21.15.16:6804/101543","heartbeat_front_addr":"172.21.15.16:6805/101543","state":["exists","up"]},{"osd":1,"uuid":"d64abb10-97f6-4b6a-b63d-cf629ae6c316","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":16,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6803","nonce":101570}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6802","nonce":101570}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6807","nonce":101570}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.16:6806","nonce":101570}]},"public_addr":"172.21.15.16:6803/101570","cluster_addr":"172.21.15.16:6802/101570","heartbeat_back_addr":"172.21.15.16:6807/101570","heartbeat_front_addr":"172.21.15.16:6806/101570","state":["exists","up"]},{"osd":2,"uuid":"02dbe754-99c6-4d7d-9374-db83799791e4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":16,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6803","nonce":101147}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6802","nonce":101147}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6807","nonce":101147}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6806","nonce":101147}]},"public_addr":"172.21.15.170:6803/101147","cluster_addr":"172.21.15.170:6802/101147","heartbeat_back_addr":"172.21.15.170:6807/101147","heartbeat_front_addr":"172.21.15.170:6806/101147","state":["exists","up"]},{"osd":3,"uuid":"aeee6307-59c7-418d-ab93-d1cf3bbf6ae9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":16,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6804","nonce":101175}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6805","nonce":101175}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6808","nonce":101175}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.170:6809","nonce":101175}]},"public_addr":"172.21.15.170:6804/101175","cluster_addr":"172.21.15.170:6805/101175","heartbeat_back_addr":"172.21.15.170:6808/101175","heartbeat_front_addr":"172.21.15.170:6809/101175","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2023-05-23T07:21:05.564 DEBUG:teuthology.orchestra.run.smithi016:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 flush_pg_stats 2023-05-23T10:30:30.968 WARNING:scrape:Ignoring malparsed backtrace: run.wait(tests.values()) , File "/home/teuthworker/src/git.ceph.com_teuthology_6fb4f0c910de3902d27909a642e8d50d731dd28c/teuthology/orchestra/run.py", line 479, in wait , proc.wait() 2023-05-23T10:30:30.969 WARNING:scrape:Ignoring malparsed backtrace: 2023-05-23T07:21:23.312 DEBUG:teuthology.orchestra.run.smithi170:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits , 2023-05-23T07:21:23.407 DEBUG:teuthology.run_tasks:Unwinding manager clock , 2023-05-23T07:21:23.417 INFO:teuthology.task.clock:Checking final clock skew... 2023-05-23T10:30:30.970 WARNING:scrape:Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_teuthology_6fb4f0c910de3902d27909a642e8d50d731dd28c/teuthology/orchestra/run.py", line 161, in wait , self._raise_for_status() , File "/home/teuthworker/src/git.ceph.com_teuthology_6fb4f0c910de3902d27909a642e8d50d731dd28c/teuthology/orchestra/run.py", line 179, in _raise_for_status 2023-05-23T10:30:30.986 INFO:scrape:Found 10 distinct failure reasons 2023-05-23T10:30:30.988 INFO:scrape:Timeout 120 running ceph --cluster ceph status --format=json 2023-05-23T10:30:30.989 INFO:scrape:2 jobs: ['7283257', '7283255'] 2023-05-23T10:30:30.989 INFO:scrape:suites intersection: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{default}', 'centos_latest', 'clusters/{fixed-2}', 'crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'crimson_qa_overrides', 'deploy/ceph', 'objectstore/bluestore', 'thrashers/default', 'thrashosds-health'] 2023-05-23T10:30:30.989 INFO:scrape:suites union: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{default}', 'centos_latest', 'clusters/{fixed-2}', 'crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'crimson_qa_overrides', 'deploy/ceph', 'objectstore/bluestore', 'thrashers/default', 'thrashosds-health', 'workloads/small-objects-localized}', 'workloads/small-objects}'] 2023-05-23T10:30:30.990 INFO:scrape: 2023-05-23T10:30:30.990 INFO:scrape:Timeout 1h running clone.client.0/qa/workunits/rados/test_python.sh --eval-attr \ 2023-05-23T10:30:30.990 INFO:scrape:1 jobs: ['7283252'] 2023-05-23T10:30:30.991 INFO:scrape:suites: ['clusters/fixed-2', 'crimson-rados/basic/{centos_latest', 'crimson_qa_overrides', 'deploy/ceph', 'tasks/rados_python}'] 2023-05-23T10:30:30.991 INFO:scrape: 2023-05-23T10:30:30.991 INFO:scrape:Timeout 120 running ceph --cluster ceph pg dump --format=json 2023-05-23T10:30:31.008 INFO:scrape:2 jobs: ['7283254', '7283251'] 2023-05-23T10:30:31.009 INFO:scrape:suites intersection: ['crimson_qa_overrides', 'deploy/ceph'] 2023-05-23T10:30:31.009 INFO:scrape:suites union: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{default}', 'centos_latest', 'clusters/fixed-1', 'clusters/{fixed-2}', 'crimson-rados/rbd/{centos_latest', 'crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'crimson_qa_overrides', 'deploy/ceph', 'objectstore/bluestore', 'tasks/rbd_lock_and_fence}', 'thrashers/default', 'thrashosds-health', 'workloads/radosbench}'] 2023-05-23T10:30:31.009 INFO:scrape: 2023-05-23T10:30:31.010 INFO:scrape:Failure: Command crashed: 'CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --balance-reads --max-ops 400000 --objects 1024 --max-in-flight 64 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 600 --op read 100 --op write 50 --op delete 50 --op snap_create 50 --op snap_remove 0 --op rollback 0 --op setattr 25 --op rmattr 25 --op copy_from 0 --op write_excl 50 --pool unique_pool_0' 2023-05-23T10:30:31.010 INFO:scrape:2 jobs: ['7283253', '7283259'] 2023-05-23T10:30:31.010 INFO:scrape:suites intersection: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{default}', 'centos_latest', 'clusters/{fixed-2}', 'crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'crimson_qa_overrides', 'deploy/ceph', 'objectstore/bluestore', 'thrashers/default', 'thrashosds-health'] 2023-05-23T10:30:31.032 INFO:scrape:suites union: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{default}', 'centos_latest', 'clusters/{fixed-2}', 'crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'crimson_qa_overrides', 'deploy/ceph', 'objectstore/bluestore', 'thrashers/default', 'thrashosds-health', 'workloads/small-objects-balanced}', 'workloads/write_fadvise_dontneed}'] 2023-05-23T10:30:31.032 INFO:scrape: 2023-05-23T10:30:31.033 INFO:scrape:Timeout 3h running clone.client.0/qa/workunits/rbd/crimson/test_crimson_librbd.sh 2023-05-23T10:30:31.033 INFO:scrape:1 jobs: ['7283247'] 2023-05-23T10:30:31.033 INFO:scrape:suites: ['clusters/fixed-1', 'crimson-rados/rbd/{centos_latest', 'crimson_qa_overrides', 'deploy/ceph', 'tasks/rbd_api_tests_old_format}'] 2023-05-23T10:30:31.033 INFO:scrape: 2023-05-23T10:30:31.034 INFO:scrape:Failure: reached maximum tries (500) after waiting for 3000 seconds 2023-05-23T10:30:31.034 INFO:scrape:2 jobs: ['7283249', '7283248'] 2023-05-23T10:30:31.040 INFO:scrape:suites intersection: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{default}', 'centos_latest', 'clusters/{fixed-2}', 'crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'crimson_qa_overrides', 'deploy/ceph', 'objectstore/bluestore', 'thrashers/default', 'thrashosds-health'] 2023-05-23T10:30:31.041 INFO:scrape:suites union: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{default}', 'centos_latest', 'clusters/{fixed-2}', 'crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'crimson_qa_overrides', 'deploy/ceph', 'objectstore/bluestore', 'thrashers/default', 'thrashosds-health', 'workloads/admin_socket_objecter_requests}', 'workloads/radosbench-high-concurrency}'] 2023-05-23T10:30:31.042 INFO:scrape: 2023-05-23T10:30:31.042 INFO:scrape:Failure: Command failed (workunit test cls/test_cls_rbd.sh) on smithi155 with status 139: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=031fa64016bfbb21467aeca1fc33541e74a27742 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 CRIMSON_COMPAT=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2023-05-23T10:30:31.042 INFO:scrape:1 jobs: ['7283250'] 2023-05-23T10:30:31.043 INFO:scrape:suites: ['clusters/fixed-1', 'crimson-rados/rbd/{centos_latest', 'crimson_qa_overrides', 'deploy/ceph', 'tasks/rbd_cls_tests}'] 2023-05-23T10:30:31.043 INFO:scrape: 2023-05-23T10:30:31.043 INFO:scrape:Timeout 3h running clone.client.0/qa/workunits/rados/test.sh 2023-05-23T10:30:31.043 INFO:scrape:1 jobs: ['7283246'] 2023-05-23T10:30:31.044 INFO:scrape:suites: ['clusters/fixed-2', 'crimson-rados/basic/{centos_latest', 'crimson_qa_overrides', 'deploy/ceph', 'tasks/rados_api_tests}'] 2023-05-23T10:30:31.044 INFO:scrape: 2023-05-23T10:30:31.044 INFO:scrape:Timeout 3h running clone.client.0/qa/workunits/rbd/test_librbd_python.sh 2023-05-23T10:30:31.051 INFO:scrape:1 jobs: ['7283258'] 2023-05-23T10:30:31.052 INFO:scrape:suites: ['clusters/fixed-1', 'crimson-rados/rbd/{centos_latest', 'crimson_qa_overrides', 'deploy/ceph', 'tasks/rbd_python_api_tests_old_format}'] 2023-05-23T10:30:31.052 INFO:scrape: 2023-05-23T10:30:31.052 INFO:scrape:Failure: Command crashed: 'CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --max-ops 4000 --objects 500 --max-in-flight 16 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 0 --op read 45 --op write 22 --op delete 10 --op write_excl 22 --pool unique_pool_0' 2023-05-23T10:30:31.052 INFO:scrape:1 jobs: ['7283256'] 2023-05-23T10:30:31.053 INFO:scrape:suites: ['clusters/fixed-2', 'crimson-rados/basic/{centos_latest', 'crimson_qa_overrides', 'deploy/ceph', 'tasks/readwrite}'] 2023-05-23T10:30:31.053 INFO:scrape: 2023-05-23T10:30:31.054 INFO:teuthology.results:Sending results to chunmei.liu@intel.com: Test Run: yingxin-2023-05-23_06:48:32-crimson-rados-wip-chunmei-multicore-os-crimson-only-distro-default-smithi ================================================================= info: http://pulpito.ceph.com/yingxin-2023-05-23_06:48:32-crimson-rados-wip-chunmei-multicore-os-crimson-only-distro-default-smithi/ logs: http://qa-proxy.ceph.com/teuthology/yingxin-2023-05-23_06:48:32-crimson-rados-wip-chunmei-multicore-os-crimson-only-distro-default-smithi/ failed: 14 dead: 0 running: 0 waiting: 0 queued: 0 passed: 0 Fail ================================================================= [7283246] crimson-rados/basic/{centos_latest clusters/fixed-2 crimson_qa_overrides deploy/ceph tasks/rados_api_tests} ----------------------------------------------------------------- time: 03:21:55 info: http://pulpito.ceph.com/yingxin-2023-05-23_06:48:32-crimson-rados-wip-chunmei-multicore-os-crimson-only-distro-default-smithi/7283246/ log: http://qa-proxy.ceph.com/teuthology/yingxin-2023-05-23_06:48:32-crimson-rados-wip-chunmei-multicore-os-crimson-only-distro-default-smithi/7283246/ sentry: https://sentry.ceph.com/organizations/ceph/?query=1df2a1aa65da42909e20c2501e746d62 Command failed (workunit test rados/test.sh) on smithi002 with status 124: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=031fa64016bfbb21467aeca1fc33541e74a27742 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 CRIMSON_COMPAT=1 adjust-ulimits ceph- coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rados/test.sh' [7283247] crimson-rados/rbd/{centos_latest clusters/fixed-1 crimson_qa_overrides deploy/ceph tasks/rbd_api_tests_old_format} ----------------------------------------------------------------- time: 03:21:42 info: http://pulpito.ceph.com/yingxin-2023-05-23_06:48:32-crimson-rados-wip-chunmei-multicore-os-crimson-only-distro-default-smithi/7283247/ log: http://qa-proxy.ceph.com/teuthology/yingxin-2023-05-23_06:48:32-crimson-rados-wip-chunmei-multicore-os-crimson-only-distro-default-smithi/7283247/ sentry: https://sentry.ceph.com/organizations/ceph/?query=f8dc305929fc44ccba0161b1ef8b6c33 Command failed (workunit test rbd/crimson/test_crimson_librbd.sh) on smithi183 with status 124: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=031fa64016bfbb21467aeca1fc33541e74a27742 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 CRIMSON_COMPAT=1 adjust-ulimits ceph- coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cep htest/clone.client.0/qa/workunits/rbd/crimson/test_crimson_librbd.sh' [7283248] crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{default} centos_latest clusters/{fixed-2} crimson_qa_overrides deploy/ceph objectstore/bluestore thrashers/default thrashosds-health workloads/admin_socket_objecter_requests} ----------------------------------------------------------------- time: 02:29:07 info: http://pulpito.ceph.com/yingxin-2023-05-23_06:48:32-crimson-rados-wip-chunmei-multicore-os-crimson-only-distro-default-smithi/7283248/ log: http://qa-proxy.ceph.com/teuthology/yingxin-2023-05-23_06:48:32-crimson-rados-wip-chunmei-multicore-os-crimson-only-distro-default-smithi/7283248/ reached maximum tries (800) after waiting for 4800 seconds [7283249] crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{default} centos_latest clusters/{fixed-2} crimson_qa_overrides deploy/ceph objectstore/bluestore thrashers/default thrashosds-health workloads/radosbench-high-concurrency} ----------------------------------------------------------------- time: 02:03:14 info: http://pulpito.ceph.com/yingxin-2023-05-23_06:48:32-crimson-rados-wip-chunmei-multicore-os-crimson-only-distro-default-smithi/7283249/ log: http://qa-proxy.ceph.com/teuthology/yingxin-2023-05-23_06:48:32-crimson-rados-wip-chunmei-multicore-os-crimson-only-distro-default-smithi/7283249/ sentry: https://sentry.ceph.com/organizations/ceph/?query=c842a454ac284419a5fdfa88e68c5335 reached maximum tries (500) after waiting for 3000 seconds [7283250] crimson-rados/rbd/{centos_latest clusters/fixed-1 crimson_qa_overrides deploy/ceph tasks/rbd_cls_tests} ----------------------------------------------------------------- time: 00:14:07 info: http://pulpito.ceph.com/yingxin-2023-05-23_06:48:32-crimson-rados-wip-chunmei-multicore-os-crimson-only-distro-default-smithi/7283250/ log: http://qa-proxy.ceph.com/teuthology/yingxin-2023-05-23_06:48:32-crimson-rados-wip-chunmei-multicore-os-crimson-only-distro-default-smithi/7283250/ sentry: https://sentry.ceph.com/organizations/ceph/?query=4b29ca14cb3b4075bd527b0b28ba955f Command failed (workunit test cls/test_cls_rbd.sh) on smithi155 with status 139: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=031fa64016bfbb21467aeca1fc33541e74a27742 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 CRIMSON_COMPAT=1 adjust-ulimits ceph- coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' [7283251] crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{default} centos_latest clusters/{fixed-2} crimson_qa_overrides deploy/ceph objectstore/bluestore thrashers/default thrashosds-health workloads/radosbench} ----------------------------------------------------------------- time: 01:13:29 info: http://pulpito.ceph.com/yingxin-2023-05-23_06:48:32-crimson-rados-wip-chunmei-multicore-os-crimson-only-distro-default-smithi/7283251/ log: http://qa-proxy.ceph.com/teuthology/yingxin-2023-05-23_06:48:32-crimson-rados-wip-chunmei-multicore-os-crimson-only-distro-default-smithi/7283251/ sentry: https://sentry.ceph.com/organizations/ceph/?query=484746557ced4eac88d45f168aee426f Command failed on smithi026 with status 124: 'sudo adjust-ulimits ceph- coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json' [7283252] crimson-rados/basic/{centos_latest clusters/fixed-2 crimson_qa_overrides deploy/ceph tasks/rados_python} ----------------------------------------------------------------- time: 01:24:07 info: http://pulpito.ceph.com/yingxin-2023-05-23_06:48:32-crimson-rados-wip-chunmei-multicore-os-crimson-only-distro-default-smithi/7283252/ log: http://qa-proxy.ceph.com/teuthology/yingxin-2023-05-23_06:48:32-crimson-rados-wip-chunmei-multicore-os-crimson-only-distro-default-smithi/7283252/ sentry: https://sentry.ceph.com/organizations/ceph/?query=fad518455a464bbcb901a53bbe5df0c6 Command failed (workunit test rados/test_python.sh) on smithi060 with status 124: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=031fa64016bfbb21467aeca1fc33541e74a27742 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 CRIMSON_COMPAT=1 adjust-ulimits ceph- coverage /home/ubuntu/cephtest/archive/coverage timeout 1h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rados/test_python.sh --eval-attr \'not (wait or tier or ec or bench or stats)\'' [7283253] crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{default} centos_latest clusters/{fixed-2} crimson_qa_overrides deploy/ceph objectstore/bluestore thrashers/default thrashosds-health workloads/small-objects-balanced} ----------------------------------------------------------------- time: 01:10:55 info: http://pulpito.ceph.com/yingxin-2023-05-23_06:48:32-crimson-rados-wip-chunmei-multicore-os-crimson-only-distro-default-smithi/7283253/ log: http://qa-proxy.ceph.com/teuthology/yingxin-2023-05-23_06:48:32-crimson-rados-wip-chunmei-multicore-os-crimson-only-distro-default-smithi/7283253/ Command crashed: 'CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --balance-reads --max-ops 400000 --objects 1024 --max-in-flight 64 --size 4000000 --min- stride-size 400000 --max-stride-size 800000 --max-seconds 600 --op read 100 --op write 50 --op delete 50 --op snap_create 50 --op snap_remove 0 --op rollback 0 --op setattr 25 --op rmattr 25 --op copy_from 0 --op write_excl 50 --pool unique_pool_0' [7283254] crimson-rados/rbd/{centos_latest clusters/fixed-1 crimson_qa_overrides deploy/ceph tasks/rbd_lock_and_fence} ----------------------------------------------------------------- time: 00:21:11 info: http://pulpito.ceph.com/yingxin-2023-05-23_06:48:32-crimson-rados-wip-chunmei-multicore-os-crimson-only-distro-default-smithi/7283254/ log: http://qa-proxy.ceph.com/teuthology/yingxin-2023-05-23_06:48:32-crimson-rados-wip-chunmei-multicore-os-crimson-only-distro-default-smithi/7283254/ sentry: https://sentry.ceph.com/organizations/ceph/?query=a7d35a99016644f29ba20be4f9c67bce Command failed on smithi189 with status 124: 'sudo adjust-ulimits ceph- coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json' [7283255] crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{default} centos_latest clusters/{fixed-2} crimson_qa_overrides deploy/ceph objectstore/bluestore thrashers/default thrashosds-health workloads/small-objects-localized} ----------------------------------------------------------------- time: 01:09:37 info: http://pulpito.ceph.com/yingxin-2023-05-23_06:48:32-crimson-rados-wip-chunmei-multicore-os-crimson-only-distro-default-smithi/7283255/ log: http://qa-proxy.ceph.com/teuthology/yingxin-2023-05-23_06:48:32-crimson-rados-wip-chunmei-multicore-os-crimson-only-distro-default-smithi/7283255/ sentry: https://sentry.ceph.com/organizations/ceph/?query=9962e63d0122406ebbea61ea0c30cbdb Command failed on smithi038 with status 124: 'sudo adjust-ulimits ceph- coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json' [7283256] crimson-rados/basic/{centos_latest clusters/fixed-2 crimson_qa_overrides deploy/ceph tasks/readwrite} ----------------------------------------------------------------- time: 00:15:53 info: http://pulpito.ceph.com/yingxin-2023-05-23_06:48:32-crimson-rados-wip-chunmei-multicore-os-crimson-only-distro-default-smithi/7283256/ log: http://qa-proxy.ceph.com/teuthology/yingxin-2023-05-23_06:48:32-crimson-rados-wip-chunmei-multicore-os-crimson-only-distro-default-smithi/7283256/ Command crashed: 'CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --max-ops 4000 --objects 500 --max-in-flight 16 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 0 --op read 45 --op write 22 --op delete 10 --op write_excl 22 --pool unique_pool_0' [7283257] crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{default} centos_latest clusters/{fixed-2} crimson_qa_overrides deploy/ceph objectstore/bluestore thrashers/default thrashosds-health workloads/small-objects} ----------------------------------------------------------------- time: 01:09:12 info: http://pulpito.ceph.com/yingxin-2023-05-23_06:48:32-crimson-rados-wip-chunmei-multicore-os-crimson-only-distro-default-smithi/7283257/ log: http://qa-proxy.ceph.com/teuthology/yingxin-2023-05-23_06:48:32-crimson-rados-wip-chunmei-multicore-os-crimson-only-distro-default-smithi/7283257/ sentry: https://sentry.ceph.com/organizations/ceph/?query=fd6c3e5b6da64a5cbeead0b328564cc8 Command failed on smithi071 with status 124: 'sudo adjust-ulimits ceph- coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json' [7283258] crimson-rados/rbd/{centos_latest clusters/fixed-1 crimson_qa_overrides deploy/ceph tasks/rbd_python_api_tests_old_format} ----------------------------------------------------------------- time: 03:14:36 info: http://pulpito.ceph.com/yingxin-2023-05-23_06:48:32-crimson-rados-wip-chunmei-multicore-os-crimson-only-distro-default-smithi/7283258/ log: http://qa-proxy.ceph.com/teuthology/yingxin-2023-05-23_06:48:32-crimson-rados-wip-chunmei-multicore-os-crimson-only-distro-default-smithi/7283258/ sentry: https://sentry.ceph.com/organizations/ceph/?query=031bf879a7434b188585aae357e90e9c Command failed (workunit test rbd/test_librbd_python.sh) on smithi192 with status 124: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=031fa64016bfbb21467aeca1fc33541e74a27742 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 CRIMSON_COMPAT=1 adjust-ulimits ceph- coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cep htest/clone.client.0/qa/workunits/rbd/test_librbd_python.sh' [7283259] crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{default} centos_latest clusters/{fixed-2} crimson_qa_overrides deploy/ceph objectstore/bluestore thrashers/default thrashosds-health workloads/write_fadvise_dontneed} ----------------------------------------------------------------- time: 00:13:25 info: http://pulpito.ceph.com/yingxin-2023-05-23_06:48:32-crimson-rados-wip-chunmei-multicore-os-crimson-only-distro-default-smithi/7283259/ log: http://qa-proxy.ceph.com/teuthology/yingxin-2023-05-23_06:48:32-crimson-rados-wip-chunmei-multicore-os-crimson-only-distro-default-smithi/7283259/ Command crashed: 'CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --write-fadvise- dontneed --max-ops 4000 --objects 500 --max-in-flight 16 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 0 --op read 100 --op write 50 --op delete 10 --op write_excl 50 --pool unique_pool_0'