2021-02-28T13:55:36.617 INFO:root:teuthology version: 1.1.0-3082387b 2021-02-28T13:55:36.619 INFO:teuthology.results:subset: '59/100000' 2021-02-28T13:55:36.619 INFO:teuthology.results:seed: '6804' 2021-02-28T14:09:49.435 INFO:root:teuthology version: 1.1.0-3082387b 2021-02-28T14:09:49.478 INFO:teuthology.results:Waiting up to 43200 seconds for tests to finish... 2021-03-01T02:10:42.490 WARNING:teuthology.results:test(s) did not finish before timeout of 43200 seconds 2021-03-01T02:10:46.225 INFO:scrape:Found 8 jobs 2021-03-01T02:10:46.233 INFO:scrape:Found 1 distinct failure reasons 2021-03-01T02:10:46.234 INFO:scrape:Dead: b'2021-03-01T02:10:45.417 INFO:teuthology.orchestra.run.gibba004.stdout:{"epoch":38,"fsid":"2a00dfc8-c06e-45b8-9205-1f870f2b418b","created":"2021-02-28T14:24:44.407501+0000","modified":"2021-03-01T00:19:53.437313+0000","last_up_change":"2021-03-01T00:19:53.437313+0000","last_in_change":"2021-02-28T14:24:48.416520+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2021-02-28T14:24:48.437115+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0\'0","target_version":"0\'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2021-02-28T14:24:53.076165+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0\'0","target_version":"0\'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2021-02-28T14:25:02.677152+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0\'0","target_version":"0\'0"},"last_change":"30","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"12dae6d2-8745-478b-8ca8-b1e3efdf8503","up":0,"in":1,"weight":1,"primary_affinity":0,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":36,"down_at":38,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.104:6800","nonce":28251}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.104:6801","nonce":28251}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.104:6804","nonce":28251}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.104:6805","nonce":28251}]},"public_addr":"172.21.2.104:6800/28251","cluster_addr":"172.21.2.104:6801/28251","heartbeat_back_addr":"172.21.2.104:6804/28251","heartbeat_front_addr":"172.21.2.104:6805/28251","state":["exists"]},{"osd":1,"uuid":"e48c9e25-7a61-4acc-bba9-a21680119845","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":30,"down_at":34,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.104:6802","nonce":28288}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.104:6803","nonce":28288}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.104:6806","nonce":28288}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.104:6807","nonce":28288}]},"public_addr":"172.21.2.104:6802/28288","cluster_addr":"172.21.2.104:6803/28288","heartbeat_back_addr":"172.21.2.104:6806/28288","heartbeat_front_addr":"172.21.2.104:6807/28288","state":["exists"]},{"osd":2,"uuid":"581d31f8-f083-486b-8426-603a606da5e8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":36,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.139:6800","nonce":28013}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.139:6801","nonce":28013}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.139:6804","nonce":28013}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.139:6805","nonce":28013}]},"public_addr":"172.21.2.139:6800/28013","cluster_addr":"172.21.2.139:6801/28013","heartbeat_back_addr":"172.21.2.139:6804/28013","heartbeat_front_addr":"172.21.2.139:6805/28013","state":["exists","up"]},{"osd":3,"uuid":"3a402bda-711b-4fa9-955c-9ae7e8205560","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":14,"up_from":30,"up_thru":34,"down_at":36,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.139:6802","nonce":29221}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.139:6803","nonce":29221}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.139:6806","nonce":29221}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.2.139:6807","nonce":29221}]},"public_addr":"172.21.2.139:6802/29221","cluster_addr":"172.21.2.139:6803/29221","heartbeat_back_addr":"172.21.2.139:6806/29221","heartbeat_front_addr":"172.21.2.139:6807/29221","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"2021-03-01T00:19:53.437313+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"2021-02-28T14:26:00.401866+0000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0.30000001192092896,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"2021-02-28T14:26:30.470789+0000","laggy_probability":0.50999999046325684,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}}' 2021-03-01T02:10:46.235 INFO:scrape:1 jobs: ['5919703'] 2021-03-01T02:10:46.235 INFO:scrape:suites: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{default}', 'centos_latest', 'clusters/{fixed-2}', 'crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'deploy/ceph', 'objectstore/bluestore', 'thrashers/default', 'thrashosds-health', 'workloads/small-objects}'] 2021-03-01T02:10:46.236 INFO:scrape: 2021-03-01T02:10:46.237 INFO:teuthology.results:Sending results to ceph-qa@ceph.io: Test Run: teuthology-2021-02-28_01:01:03-crimson-rados-master-distro-basic-gibba ================================================================= info: http://pulpito.ceph.com/teuthology-2021-02-28_01:01:03-crimson-rados-master-distro-basic-gibba/ logs: http://qa-proxy.ceph.com/teuthology/teuthology-2021-02-28_01:01:03-crimson-rados-master-distro-basic-gibba/ failed: 0 dead: 0 running: 1 waiting: 0 queued: 0 passed: 7 Running ================================================================= [5919703] crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{default} centos_latest clusters/{fixed-2} deploy/ceph objectstore/bluestore thrashers/default thrashosds-health workloads/small-objects} info: http://pulpito.ceph.com/teuthology-2021-02-28_01:01:03-crimson-rados-master-distro-basic-gibba/5919703/ Pass ================================================================= [5919701] crimson-rados/basic/{centos_latest ceph clusters/fixed-2 tasks/rados_api_tests} time: 01:12:23 info: http://pulpito.ceph.com/teuthology-2021-02-28_01:01:03-crimson-rados-master-distro-basic-gibba/5919701/ [5919702] crimson-rados/rbd/{centos_latest ceph clusters/fixed-1 tasks/rbd_api_tests_old_format} time: 00:59:45 info: http://pulpito.ceph.com/teuthology-2021-02-28_01:01:03-crimson-rados-master-distro-basic-gibba/5919702/ [5919704] crimson-rados/rbd/{centos_latest ceph clusters/fixed-1 tasks/rbd_cls_tests} time: 00:58:24 info: http://pulpito.ceph.com/teuthology-2021-02-28_01:01:03-crimson-rados-master-distro-basic-gibba/5919704/ [5919705] crimson-rados/basic/{centos_latest ceph clusters/fixed-2 tasks/rados_python} time: 01:02:42 info: http://pulpito.ceph.com/teuthology-2021-02-28_01:01:03-crimson-rados-master-distro-basic-gibba/5919705/ [5919706] crimson-rados/rbd/{centos_latest ceph clusters/fixed-1 tasks/rbd_lock_and_fence} time: 00:56:37 info: http://pulpito.ceph.com/teuthology-2021-02-28_01:01:03-crimson-rados-master-distro-basic-gibba/5919706/ [5919707] crimson-rados/basic/{centos_latest ceph clusters/fixed-2 tasks/readwrite} time: 00:10:21 info: http://pulpito.ceph.com/teuthology-2021-02-28_01:01:03-crimson-rados-master-distro-basic-gibba/5919707/ [5919708] crimson-rados/rbd/{centos_latest ceph clusters/fixed-1 tasks/rbd_python_api_tests_old_format} time: 01:02:47 info: http://pulpito.ceph.com/teuthology-2021-02-28_01:01:03-crimson-rados-master-distro-basic-gibba/5919708/