2021-02-11T14:38:27.593 INFO:root:teuthology version: 1.1.0-f0562195
2021-02-11T14:38:27.594 INFO:teuthology.results:Waiting up to 43200 seconds for tests to finish...
2021-02-12T02:38:40.208 WARNING:teuthology.results:test(s) did not finish before timeout of 43200 seconds
2021-02-12T02:38:44.315 INFO:scrape:Found 6 jobs
2021-02-12T02:38:45.335 INFO:scrape:Found 5 distinct failure reasons
2021-02-12T02:38:45.336 INFO:scrape:Dead: b'2021-02-12T02:38:41.564 INFO:teuthology.orchestra.run.smithi098.stdout:{"epoch":5793,"fsid":"4effac79-2219-4a97-8362-70fefa87322c","created":"2021-02-11T14:54:15.222932+0000","modified":"2021-02-12T02:38:18.859966+0000","last_up_change":"2021-02-11T15:22:54.587149+0000","last_in_change":"2021-02-11T15:26:56.278028+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":6,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2021-02-11T14:54:19.556536+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":111,"pg_placement_num":111,"pg_placement_num_target":111,"pg_num_target":111,"pg_num_pending":111,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0\'0","target_version":"0\'0"},"last_change":"1578","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"1423","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2021-02-11T14:54:28.218948+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":98,"pg_placement_num":98,"pg_placement_num_target":98,"pg_num_target":98,"pg_num_pending":98,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0\'0","target_version":"0\'0"},"last_change":"1577","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"900","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"a2734530-e77b-42d2-bf31-006b2e0949bf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":15,"last_clean_end":431,"up_from":489,"up_thru":1595,"down_at":432,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6816","nonce":45920},{"type":"v1","addr":"172.21.15.98:6817","nonce":45920}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6818","nonce":45920},{"type":"v1","addr":"172.21.15.98:6819","nonce":45920}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6822","nonce":45920},{"type":"v1","addr":"172.21.15.98:6823","nonce":45920}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6820","nonce":45920},{"type":"v1","addr":"172.21.15.98:6821","nonce":45920}]},"public_addr":"172.21.15.98:6817/45920","cluster_addr":"172.21.15.98:6819/45920","heartbeat_back_addr":"172.21.15.98:6823/45920","heartbeat_front_addr":"172.21.15.98:6821/45920","state":["exists","up"]},{"osd":1,"uuid":"d67354c1-64e9-4747-afc1-a73764c12868","up":0,"in":0,"weight":0,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":750,"down_at":756,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6800","nonce":15424},{"type":"v1","addr":"172.21.15.98:6801","nonce":15424}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6802","nonce":15424},{"type":"v1","addr":"172.21.15.98:6803","nonce":15424}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6806","nonce":15424},{"type":"v1","addr":"172.21.15.98:6807","nonce":15424}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6804","nonce":15424},{"type":"v1","addr":"172.21.15.98:6805","nonce":15424}]},"public_addr":"172.21.15.98:6801/15424","cluster_addr":"172.21.15.98:6803/15424","heartbeat_back_addr":"172.21.15.98:6807/15424","heartbeat_front_addr":"172.21.15.98:6805/15424","state":["autoout","exists"]},{"osd":2,"uuid":"e995df9b-bc2c-4abe-8f84-cbb5d041c3ed","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":1595,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6808","nonce":15431},{"type":"v1","addr":"172.21.15.98:6809","nonce":15431}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6810","nonce":15431},{"type":"v1","addr":"172.21.15.98:6811","nonce":15431}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6814","nonce":15431},{"type":"v1","addr":"172.21.15.98:6815","nonce":15431}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6812","nonce":15431},{"type":"v1","addr":"172.21.15.98:6813","nonce":15431}]},"public_addr":"172.21.15.98:6809/15431","cluster_addr":"172.21.15.98:6811/15431","heartbeat_back_addr":"172.21.15.98:6815/15431","heartbeat_front_addr":"172.21.15.98:6813/15431","state":["exists","up"]},{"osd":3,"uuid":"170787a7-5491-499b-845c-01500cb1a254","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":227,"last_clean_end":574,"up_from":650,"up_thru":1598,"down_at":575,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6824","nonce":57749},{"type":"v1","addr":"172.21.15.98:6825","nonce":57749}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6826","nonce":57749},{"type":"v1","addr":"172.21.15.98:6827","nonce":57749}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6830","nonce":57749},{"type":"v1","addr":"172.21.15.98:6831","nonce":57749}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.98:6828","nonce":57749},{"type":"v1","addr":"172.21.15.98:6829","nonce":57749}]},"public_addr":"172.21.15.98:6825/57749","cluster_addr":"172.21.15.98:6827/57749","heartbeat_back_addr":"172.21.15.98:6831/57749","heartbeat_front_addr":"172.21.15.98:6829/57749","state":["exists","up"]},{"osd":4,"uuid":"55977ff8-4720-44db-b412-d62b690d209c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":1598,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6802","nonce":15426},{"type":"v1","addr":"172.21.15.172:6803","nonce":15426}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6804","nonce":15426},{"type":"v1","addr":"172.21.15.172:6805","nonce":15426}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6808","nonce":15426},{"type":"v1","addr":"172.21.15.172:6809","nonce":15426}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6806","nonce":15426},{"type":"v1","addr":"172.21.15.172:6807","nonce":15426}]},"public_addr":"172.21.15.172:6803/15426","cluster_addr":"172.21.15.172:6805/15426","heartbeat_back_addr":"172.21.15.172:6809/15426","heartbeat_front_addr":"172.21.15.172:6807/15426","state":["exists","up"]},{"osd":5,"uuid":"04505810-bc2f-40ae-8123-28ec76ee8dde","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":1586,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6818","nonce":15427},{"type":"v1","addr":"172.21.15.172:6819","nonce":15427}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6820","nonce":15427},{"type":"v1","addr":"172.21.15.172:6821","nonce":15427}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6824","nonce":15427},{"type":"v1","addr":"172.21.15.172:6825","nonce":15427}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6822","nonce":15427},{"type":"v1","addr":"172.21.15.172:6823","nonce":15427}]},"public_addr":"172.21.15.172:6819/15427","cluster_addr":"172.21.15.172:6821/15427","heartbeat_back_addr":"172.21.15.172:6825/15427","heartbeat_front_addr":"172.21.15.172:6823/15427","state":["exists","up"]},{"osd":6,"uuid":"b9cceb48-fbf8-492f-9171-d77254e3f9af","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":1584,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6810","nonce":15428},{"type":"v1","addr":"172.21.15.172:6811","nonce":15428}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6812","nonce":15428},{"type":"v1","addr":"172.21.15.172:6813","nonce":15428}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6816","nonce":15428},{"type":"v1","addr":"172.21.15.172:6817","nonce":15428}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6814","nonce":15428},{"type":"v1","addr":"172.21.15.172:6815","nonce":15428}]},"public_addr":"172.21.15.172:6811/15428","cluster_addr":"172.21.15.172:6813/15428","heartbeat_back_addr":"172.21.15.172:6817/15428","heartbeat_front_addr":"172.21.15.172:6815/15428","state":["exists","up"]},{"osd":7,"uuid":"30c888fc-b6b8-4559-b1d7-33d8b753653f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":15,"last_clean_end":1329,"up_from":1410,"up_thru":1602,"down_at":1331,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6826","nonce":39417},{"type":"v1","addr":"172.21.15.172:6827","nonce":39417}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6828","nonce":39417},{"type":"v1","addr":"172.21.15.172:6829","nonce":39417}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6832","nonce":39417},{"type":"v1","addr":"172.21.15.172:6833","nonce":39417}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.172:6830","nonce":39417},{"type":"v1","addr":"172.21.15.172:6831","nonce":39417}]},"public_addr":"172.21.15.172:6827/39417","cluster_addr":"172.21.15.172:6829/39417","heartbeat_back_addr":"172.21.15.172:6833/39417","heartbeat_front_addr":"172.21.15.172:6831/39417","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-12T02:38:02.548840+0000","dead_epoch":432},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":65536,"last_purged_snaps_scrub":"2021-02-11T15:09:29.341776+0000","dead_epoch":756},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-12T02:37:55.958822+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-12T02:38:02.645828+0000","dead_epoch":575},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-12T02:37:56.702677+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-12T02:38:01.638571+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-12T02:37:58.982789+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-12T02:38:18.203560+0000","dead_epoch":1331}],"pg_upmap":[{"pgid":"1.1e","osds":[1,3]},{"pgid":"1.2c","osds":[6,0]},{"pgid":"2.12","osds":[7,6]},{"pgid":"2.3f","osds":[2,5]}],"pg_upmap_items":[{"pgid":"1.2","mappings":[{"from":0,"to":4}]},{"pgid":"1.2b","mappings":[{"from":0,"to":6}]},{"pgid":"1.32","mappings":[{"from":5,"to":4}]},{"pgid":"2.0","mappings":[{"from":7,"to":2}]},{"pgid":"2.4","mappings":[{"from":7,"to":2}]},{"pgid":"2.1f","mappings":[{"from":7,"to":2}]},{"pgid":"2.20","mappings":[{"from":7,"to":2}]},{"pgid":"2.26","mappings":[{"from":7,"to":2}]},{"pgid":"2.50","mappings":[{"from":7,"to":2}]},{"pgid":"2.60","mappings":[{"from":7,"to":2}]}],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"clay42profile":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","d":"5","k":"4","m":"2","name":"clay42profile","plugin":"clay","technique":"reed_sol_van"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":692,"length":1},{"begin":694,"length":1},{"begin":698,"length":2},{"begin":703,"length":2},{"begin":709,"length":1},{"begin":712,"length":1},{"begin":716,"length":3},{"begin":720,"length":1},{"begin":722,"length":1},{"begin":725,"length":3},{"begin":730,"length":1},{"begin":734,"length":1},{"begin":739,"length":1},{"begin":749,"length":2},{"begin":756,"length":1},{"begin":763,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}}'
2021-02-12T02:38:45.336 INFO:scrape:1 jobs: ['5874666']
2021-02-12T02:38:45.336 INFO:scrape:suites: ['clusters/{fixed-2', 'fast/normal', 'mon_election/classic', 'msgr-failures/fastclose', 'objectstore/bluestore-comp-zlib', 'openstack}', 'rados', 'rados:thrash-erasure-code/{ceph', 'recovery-overrides/{more-async-partial-recovery}', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/morepggrow', 'thrashosds-health', 'workloads/ec-rados-plugin=clay-k=4-m=2}']
2021-02-12T02:38:45.336 INFO:scrape:
2021-02-12T02:38:45.336 INFO:scrape:Dead: b'2021-02-12T02:38:44.329 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 30 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-osd.3.asok dump_blocked_ops'
2021-02-12T02:38:45.336 INFO:scrape:1 jobs: ['5874667']
2021-02-12T02:38:45.336 INFO:scrape:suites: ['clusters/{fixed-2', 'fast/normal', 'mon_election/classic', 'msgr-failures/fastclose', 'objectstore/bluestore-comp-zlib', 'openstack}', 'rados', 'rados:thrash-erasure-code/{ceph', 'recovery-overrides/{more-async-partial-recovery}', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/morepggrow', 'thrashosds-health', 'workloads/ec-rados-plugin=clay-k=4-m=2}']
2021-02-12T02:38:45.337 INFO:scrape:
2021-02-12T02:38:45.337 INFO:scrape:Dead: b'2021-02-12T02:38:42.023 INFO:teuthology.orchestra.run.smithi057.stdout:{"epoch":5231,"fsid":"f9bb4c67-5594-4c42-b989-df584b3b4dab","created":"2021-02-11T14:55:09.731471+0000","modified":"2021-02-12T02:38:37.879804+0000","last_up_change":"2021-02-11T15:18:22.322329+0000","last_in_change":"2021-02-11T15:29:25.455798+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2021-02-11T14:55:13.846263+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":31,"pg_placement_num":31,"pg_placement_num_target":31,"pg_num_target":31,"pg_num_pending":31,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0\'0","target_version":"0\'0"},"last_change":"1073","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"877","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2021-02-11T14:55:53.158581+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":88,"pg_placement_num":88,"pg_placement_num_target":88,"pg_num_target":88,"pg_num_pending":88,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0\'0","target_version":"0\'0"},"last_change":"1074","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"1012","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"39bd5543-4eac-4ebe-ab7c-2646edce70d5","up":0,"in":0,"weight":0,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":999,"down_at":1004,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.57:6810","nonce":15379},{"type":"v1","addr":"172.21.15.57:6813","nonce":15379}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.57:6816","nonce":15379},{"type":"v1","addr":"172.21.15.57:6819","nonce":15379}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.57:6826","nonce":15379},{"type":"v1","addr":"172.21.15.57:6828","nonce":15379}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.57:6820","nonce":15379},{"type":"v1","addr":"172.21.15.57:6823","nonce":15379}]},"public_addr":"172.21.15.57:6813/15379","cluster_addr":"172.21.15.57:6819/15379","heartbeat_back_addr":"172.21.15.57:6828/15379","heartbeat_front_addr":"172.21.15.57:6823/15379","state":["autoout","exists"]},{"osd":1,"uuid":"823e284a-bcde-4750-8c73-4d64b8ffe48b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":1145,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.57:6809","nonce":15377},{"type":"v1","addr":"172.21.15.57:6812","nonce":15377}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.57:6815","nonce":15377},{"type":"v1","addr":"172.21.15.57:6818","nonce":15377}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.57:6829","nonce":15377},{"type":"v1","addr":"172.21.15.57:6831","nonce":15377}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.57:6822","nonce":15377},{"type":"v1","addr":"172.21.15.57:6825","nonce":15377}]},"public_addr":"172.21.15.57:6812/15377","cluster_addr":"172.21.15.57:6818/15377","heartbeat_back_addr":"172.21.15.57:6831/15377","heartbeat_front_addr":"172.21.15.57:6825/15377","state":["exists","up"]},{"osd":2,"uuid":"de1f7f65-6bff-47b1-a881-d835f4b47dc0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":1154,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.57:6808","nonce":15395},{"type":"v1","addr":"172.21.15.57:6811","nonce":15395}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.57:6814","nonce":15395},{"type":"v1","addr":"172.21.15.57:6817","nonce":15395}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.57:6827","nonce":15395},{"type":"v1","addr":"172.21.15.57:6830","nonce":15395}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.57:6821","nonce":15395},{"type":"v1","addr":"172.21.15.57:6824","nonce":15395}]},"public_addr":"172.21.15.57:6811/15395","cluster_addr":"172.21.15.57:6817/15395","heartbeat_back_addr":"172.21.15.57:6830/15395","heartbeat_front_addr":"172.21.15.57:6824/15395","state":["exists","up"]},{"osd":3,"uuid":"299f8017-94d3-4c88-aca4-7f8edce6dbd9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":1145,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.57:6800","nonce":15378},{"type":"v1","addr":"172.21.15.57:6801","nonce":15378}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.57:6802","nonce":15378},{"type":"v1","addr":"172.21.15.57:6803","nonce":15378}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.57:6806","nonce":15378},{"type":"v1","addr":"172.21.15.57:6807","nonce":15378}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.57:6804","nonce":15378},{"type":"v1","addr":"172.21.15.57:6805","nonce":15378}]},"public_addr":"172.21.15.57:6801/15378","cluster_addr":"172.21.15.57:6803/15378","heartbeat_back_addr":"172.21.15.57:6807/15378","heartbeat_front_addr":"172.21.15.57:6805/15378","state":["exists","up"]},{"osd":4,"uuid":"32292eb9-64a6-44b9-a3be-61c989e985f2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":15,"last_clean_end":915,"up_from":1037,"up_thru":1145,"down_at":917,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.95:6826","nonce":26343},{"type":"v1","addr":"172.21.15.95:6827","nonce":26343}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.95:6828","nonce":26343},{"type":"v1","addr":"172.21.15.95:6829","nonce":26343}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.95:6832","nonce":26343},{"type":"v1","addr":"172.21.15.95:6833","nonce":26343}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.95:6830","nonce":26343},{"type":"v1","addr":"172.21.15.95:6831","nonce":26343}]},"public_addr":"172.21.15.95:6827/26343","cluster_addr":"172.21.15.95:6829/26343","heartbeat_back_addr":"172.21.15.95:6833/26343","heartbeat_front_addr":"172.21.15.95:6831/26343","state":["exists","up"]},{"osd":5,"uuid":"939b98de-af4f-4b85-b4e6-5c6482c61566","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":1153,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.95:6816","nonce":15470},{"type":"v1","addr":"172.21.15.95:6818","nonce":15470}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.95:6820","nonce":15470},{"type":"v1","addr":"172.21.15.95:6821","nonce":15470}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.95:6824","nonce":15470},{"type":"v1","addr":"172.21.15.95:6825","nonce":15470}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.95:6822","nonce":15470},{"type":"v1","addr":"172.21.15.95:6823","nonce":15470}]},"public_addr":"172.21.15.95:6818/15470","cluster_addr":"172.21.15.95:6821/15470","heartbeat_back_addr":"172.21.15.95:6825/15470","heartbeat_front_addr":"172.21.15.95:6823/15470","state":["exists","up"]},{"osd":6,"uuid":"4791cad7-4a27-485b-b961-df387271c555","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":1143,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.95:6802","nonce":15469},{"type":"v1","addr":"172.21.15.95:6803","nonce":15469}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.95:6804","nonce":15469},{"type":"v1","addr":"172.21.15.95:6805","nonce":15469}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.95:6808","nonce":15469},{"type":"v1","addr":"172.21.15.95:6809","nonce":15469}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.95:6806","nonce":15469},{"type":"v1","addr":"172.21.15.95:6807","nonce":15469}]},"public_addr":"172.21.15.95:6803/15469","cluster_addr":"172.21.15.95:6805/15469","heartbeat_back_addr":"172.21.15.95:6809/15469","heartbeat_front_addr":"172.21.15.95:6807/15469","state":["exists","up"]},{"osd":7,"uuid":"17395185-fb81-4973-8b80-65d12064e4e7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":1145,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.95:6810","nonce":15471},{"type":"v1","addr":"172.21.15.95:6811","nonce":15471}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.95:6812","nonce":15471},{"type":"v1","addr":"172.21.15.95:6813","nonce":15471}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.95:6817","nonce":15471},{"type":"v1","addr":"172.21.15.95:6819","nonce":15471}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.95:6814","nonce":15471},{"type":"v1","addr":"172.21.15.95:6815","nonce":15471}]},"public_addr":"172.21.15.95:6811/15471","cluster_addr":"172.21.15.95:6813/15471","heartbeat_back_addr":"172.21.15.95:6819/15471","heartbeat_front_addr":"172.21.15.95:6815/15471","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":65536,"last_purged_snaps_scrub":"2021-02-11T15:17:29.037321+0000","dead_epoch":1004},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-12T02:38:00.813441+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-12T02:38:15.364191+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-12T02:38:01.913335+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-12T02:38:37.368554+0000","dead_epoch":917},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-12T02:38:12.374117+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-12T02:37:57.120875+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2021-02-12T02:38:01.427882+0000","dead_epoch":0}],"pg_upmap":[{"pgid":"1.8","osds":[0,3]},{"pgid":"2.c","osds":[5,2]},{"pgid":"2.35","osds":[7,3]}],"pg_upmap_items":[{"pgid":"1.12","mappings":[{"from":4,"to":2}]},{"pgid":"1.1b","mappings":[{"from":1,"to":7}]},{"pgid":"1.1e","mappings":[{"from":4,"to":7}]},{"pgid":"2.1","mappings":[{"from":5,"to":2}]},{"pgid":"2.5","mappings":[{"from":3,"to":7}]},{"pgid":"2.7","mappings":[{"from":3,"to":7}]},{"pgid":"2.1a","mappings":[{"from":3,"to":7}]},{"pgid":"2.31","mappings":[{"from":1,"to":2}]},{"pgid":"2.32","mappings":[{"from":3,"to":7}]},{"pgid":"2.34","mappings":[{"from":3,"to":7}]},{"pgid":"2.4d","mappings":[{"from":1,"to":2}]},{"pgid":"2.51","mappings":[{"from":3,"to":7}]},{"pgid":"2.53","mappings":[{"from":1,"to":7}]},{"pgid":"2.54","mappings":[{"from":1,"to":2}]},{"pgid":"2.55","mappings":[{"from":1,"to":2}]}],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"clay42profile":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","d":"5","k":"4","m":"2","name":"clay42profile","plugin":"clay","technique":"reed_sol_van"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":567,"length":3},{"begin":572,"length":2},{"begin":577,"length":5},{"begin":583,"length":2},{"begin":586,"length":1},{"begin":589,"length":2},{"begin":593,"length":4},{"begin":600,"length":1},{"begin":602,"length":1},{"begin":605,"length":1},{"begin":608,"length":2},{"begin":612,"length":3},{"begin":616,"length":1},{"begin":619,"length":1},{"begin":621,"length":2},{"begin":625,"length":3},{"begin":629,"length":1},{"begin":631,"length":2},{"begin":634,"length":1},{"begin":638,"length":1},{"begin":641,"length":2},{"begin":644,"length":1},{"begin":646,"length":1},{"begin":648,"length":2},{"begin":653,"length":3},{"begin":658,"length":2},{"begin":663,"length":3},{"begin":667,"length":1},{"begin":671,"length":2},{"begin":674,"length":1},{"begin":680,"length":1},{"begin":682,"length":1},{"begin":687,"length":1},{"begin":691,"length":1},{"begin":695,"length":1},{"begin":698,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}}'
2021-02-12T02:38:45.337 INFO:scrape:1 jobs: ['5874668']
2021-02-12T02:38:45.337 INFO:scrape:suites: ['clusters/{fixed-2', 'fast/normal', 'mon_election/classic', 'msgr-failures/fastclose', 'objectstore/bluestore-comp-zlib', 'openstack}', 'rados', 'rados:thrash-erasure-code/{ceph', 'recovery-overrides/{more-async-partial-recovery}', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/morepggrow', 'thrashosds-health', 'workloads/ec-rados-plugin=clay-k=4-m=2}']
2021-02-12T02:38:45.337 INFO:scrape:
2021-02-12T02:38:45.337 INFO:scrape:Dead: b'2021-02-12T02:38:44.640 DEBUG:teuthology.orchestra.run.smithi094:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 30 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-osd.5.asok dump_blocked_ops'
2021-02-12T02:38:45.337 INFO:scrape:1 jobs: ['5874664']
2021-02-12T02:38:45.337 INFO:scrape:suites: ['clusters/{fixed-2', 'fast/normal', 'mon_election/classic', 'msgr-failures/fastclose', 'objectstore/bluestore-comp-zlib', 'openstack}', 'rados', 'rados:thrash-erasure-code/{ceph', 'recovery-overrides/{more-async-partial-recovery}', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/morepggrow', 'thrashosds-health', 'workloads/ec-rados-plugin=clay-k=4-m=2}']
2021-02-12T02:38:45.338 INFO:scrape:
2021-02-12T02:38:45.338 INFO:scrape:Dead: b'AttributeError: keep_logs'
2021-02-12T02:38:45.338 INFO:scrape:1 jobs: ['5874669']
2021-02-12T02:38:45.338 INFO:scrape:suites: ['clusters/{fixed-2', 'fast/normal', 'mon_election/classic', 'msgr-failures/fastclose', 'objectstore/bluestore-comp-zlib', 'openstack}', 'rados', 'rados:thrash-erasure-code/{ceph', 'recovery-overrides/{more-async-partial-recovery}', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/morepggrow', 'thrashosds-health', 'workloads/ec-rados-plugin=clay-k=4-m=2}']
2021-02-12T02:38:45.338 INFO:scrape: