2022-09-08T10:09:39.398 INFO:root:teuthology version: 1.1.1.dev418+gb58f2c18
2022-09-08T10:09:39.400 INFO:teuthology.results:seed: '2336'
2022-09-08T10:10:41.072 INFO:root:teuthology version: 1.1.1.dev418+gb58f2c18
2022-09-08T10:10:41.076 INFO:teuthology.results:Waiting up to 43200 seconds for tests to finish...
2022-09-08T22:11:36.615 WARNING:teuthology.results:test(s) did not finish before timeout of 43200 seconds
2022-09-08T22:11:36.790 INFO:scrape:Found 2 jobs
2022-09-08T22:11:36.937 INFO:scrape:Found 2 distinct failure reasons
2022-09-08T22:11:36.937 INFO:scrape:Dead: b"2022-09-08T22:09:55.637 DEBUG:teuthology.orchestra.run.smithi047:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false"
2022-09-08T22:11:36.938 INFO:scrape:1 jobs: ['7020184']
2022-09-08T22:11:36.938 INFO:scrape:suites: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{default}', 'centos_latest', 'clusters/{fixed-2}', 'crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'deploy/ceph', 'objectstore/bluestore', 'thrashers/default', 'thrashosds-health', 'workloads/small-objects}']
2022-09-08T22:11:36.939 INFO:scrape:
2022-09-08T22:11:36.939 INFO:scrape:Dead: b'2022-09-08T22:11:35.260 INFO:teuthology.orchestra.run.smithi163.stdout:{"epoch":52,"fsid":"6cef2ec6-3bba-4fdf-bdff-37ba87657acb","created":"2022-09-08T10:25:22.080936+0000","modified":"2022-09-08T10:36:40.187463+0000","last_up_change":"2022-09-08T10:26:21.081411+0000","last_in_change":"2022-09-08T10:36:22.136782+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-09-08T10:25:27.791782+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0\'0","target_version":"0\'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-09-08T10:25:31.619961+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0\'0","target_version":"0\'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-09-08T10:25:38.397410+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0\'0","target_version":"0\'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"1f6373fe-3144-4c5c-bc75-0e3b1d7880f0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6801","nonce":38291}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6800","nonce":38291}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6804","nonce":38291}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6805","nonce":38291}]},"public_addr":"172.21.15.163:6801/38291","cluster_addr":"172.21.15.163:6800/38291","heartbeat_back_addr":"172.21.15.163:6804/38291","heartbeat_front_addr":"172.21.15.163:6805/38291","state":["exists","up"]},{"osd":1,"uuid":"760a5a11-a9eb-4b36-888b-0b3055fff5af","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6802","nonce":38318}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6803","nonce":38318}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6806","nonce":38318}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.163:6807","nonce":38318}]},"public_addr":"172.21.15.163:6802/38318","cluster_addr":"172.21.15.163:6803/38318","heartbeat_back_addr":"172.21.15.163:6806/38318","heartbeat_front_addr":"172.21.15.163:6807/38318","state":["exists","up"]},{"osd":2,"uuid":"07f39642-1a4f-4f2e-8ac2-9ffcba4f9472","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6801","nonce":37995}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6800","nonce":37995}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6805","nonce":37995}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6804","nonce":37995}]},"public_addr":"172.21.15.164:6801/37995","cluster_addr":"172.21.15.164:6800/37995","heartbeat_back_addr":"172.21.15.164:6805/37995","heartbeat_front_addr":"172.21.15.164:6804/37995","state":["exists","up"]},{"osd":3,"uuid":"c3dd960b-5c1e-42a4-86ec-967bd9a2ba8a","up":0,"in":0,"weight":0,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":15,"down_at":31,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6803","nonce":38024}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6802","nonce":38024}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6806","nonce":38024}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6807","nonce":38024}]},"public_addr":"172.21.15.164:6803/38024","cluster_addr":"172.21.15.164:6802/38024","heartbeat_back_addr":"172.21.15.164:6806/38024","heartbeat_front_addr":"172.21.15.164:6807/38024","state":["autoout","exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2022-09-08T10:26:21.081411+0000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":65536,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}}'
2022-09-08T22:11:36.940 INFO:scrape:1 jobs: ['7020185']
2022-09-08T22:11:36.940 INFO:scrape:suites: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{default}', 'centos_latest', 'clusters/{fixed-2}', 'crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'deploy/ceph', 'objectstore/bluestore', 'thrashers/default', 'thrashosds-health', 'workloads/small-objects}']
2022-09-08T22:11:36.941 INFO:scrape: