Found 12 jobs Ignoring malparsed backtrace: ceph version 18.0.0-944-g03825a80 (03825a807f21359884cfd3f5cd4720b3dc4a0c78) reef (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7f1958a50bee] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x1b0) [0x55b2c56bc0ca] Ignoring malparsed backtrace: 2022-11-27T16:24:52.761 INFO:teuthology.orchestra.run.smithi012.stdout: , 2022-11-27T16:24:52.761 INFO:teuthology.orchestra.run.smithi012.stdout:{"epoch":132,"fsid":"456758e0-0c99-4b3c-96eb-df14daec9506","created":"2022-11-27T15:54:12.625143+0000","modified":"2022-11-27T16:24:52.053038+0000","last_up_change":"2022-11-27T15:54:18.768155+0000","last_in_change":"2022-11-27T15:54:14.002191+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-11-27T15:54:20.223835+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-11-27T15:54:22.296641+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-11-27T15:54:29.448707+0000","flags":129,"flags_names":"hashpspool,write_fadvise_dontneed","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"9f1b2568-2a6e-4094-8174-af40beb33c51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6800","nonce":103924}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6801","nonce":103924}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6804","nonce":103924}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6805","nonce":103924}]},"public_addr":"172.21.15.12:6800/103924","cluster_addr":"172.21.15.12:6801/103924","heartbeat_back_addr":"172.21.15.12:6804/103924","heartbeat_front_addr":"172.21.15.12:6805/103924","state":["exists","up"]},{"osd":1,"uuid":"7d59443e-6839-4999-93a6-fe70f05a6c70","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6802","nonce":103951}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6803","nonce":103951}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6807","nonce":103951}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6806","nonce":103951}]},"public_addr":"172.21.15.12:6802/103951","cluster_addr":"172.21.15.12:6803/103951","heartbeat_back_addr":"172.21.15.12:6807/103951","heartbeat_front_addr":"172.21.15.12:6806/103951","state":["exists","up"]},{"osd":2,"uuid":"d3df7ea4-4b95-4df0-b8c0-b10295e8b5f5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.197:6801","nonce":103508}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.197:6800","nonce":103508}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.197:6804","nonce":103508}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.197:6805","nonce":103508}]},"public_addr":"172.21.15.197:6801/103508","cluster_addr":"172.21.15.197:6800/103508","heartbeat_back_addr":"172.21.15.197:6804/103508","heartbeat_front_addr":"172.21.15.197:6805/103508","state":["exists","up"]},{"osd":3,"uuid":"13666469-a3ba-4f90-8bdf-db9967e5672a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.197:6803","nonce":103512}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.197:6802","nonce":103512}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.197:6807","nonce":103512}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.197:6806","nonce":103512}]},"public_addr":"172.21.15.197:6803/103512","cluster_addr":"172.21.15.197:6802/103512","heartbeat_back_addr":"172.21.15.197:6807/103512","heartbeat_front_addr":"172.21.15.197:6806/103512","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2022-11-27T16:24:52.775 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 flush_pg_stats Ignoring malparsed backtrace: 2022-11-27T16:25:04.270 INFO:tasks.ceph.osd.1:Stopped , 2022-11-27T16:25:04.271 DEBUG:tasks.ceph.osd.2:waiting for process to exit , 2022-11-27T16:25:04.272 INFO:teuthology.orchestra.run:waiting for 300 Ignoring malparsed backtrace: 2022-11-27T16:25:32.576 INFO:teuthology.orchestra.run.smithi012.stdout:^- hv01.front.sepia.ceph.com 3 6 377 36 +87us[ +87us] +/- 121ms , 2022-11-27T16:25:32.576 INFO:teuthology.orchestra.run.smithi012.stdout:^* hv02.front.sepia.ceph.com 2 6 377 42 -241us[ -240us] +/- 46ms , 2022-11-27T16:25:32.576 INFO:teuthology.orchestra.run.smithi012.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 38 +488us[ +488us] +/- 71ms Ignoring malparsed backtrace: remote_path = remote.sh(['which', dump_program]).rstrip() , File "/home/teuthworker/src/git.ceph.com_teuthology_4da97cf64e542f347ec47b7bdbe5eca99759f9b7/teuthology/orchestra/remote.py", line 96, in sh , proc = self.run(**kwargs) Ignoring malparsed backtrace: ceph version 18.0.0-944-g03825a80 (03825a807f21359884cfd3f5cd4720b3dc4a0c78) reef (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7f1958a50bee] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x1b0) [0x55b2c56bc0ca] Ignoring malparsed backtrace: 2022-11-27T16:24:52.761 INFO:teuthology.orchestra.run.smithi012.stdout: , 2022-11-27T16:24:52.761 INFO:teuthology.orchestra.run.smithi012.stdout:{"epoch":132,"fsid":"456758e0-0c99-4b3c-96eb-df14daec9506","created":"2022-11-27T15:54:12.625143+0000","modified":"2022-11-27T16:24:52.053038+0000","last_up_change":"2022-11-27T15:54:18.768155+0000","last_in_change":"2022-11-27T15:54:14.002191+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-11-27T15:54:20.223835+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-11-27T15:54:22.296641+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-11-27T15:54:29.448707+0000","flags":129,"flags_names":"hashpspool,write_fadvise_dontneed","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"9f1b2568-2a6e-4094-8174-af40beb33c51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6800","nonce":103924}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6801","nonce":103924}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6804","nonce":103924}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6805","nonce":103924}]},"public_addr":"172.21.15.12:6800/103924","cluster_addr":"172.21.15.12:6801/103924","heartbeat_back_addr":"172.21.15.12:6804/103924","heartbeat_front_addr":"172.21.15.12:6805/103924","state":["exists","up"]},{"osd":1,"uuid":"7d59443e-6839-4999-93a6-fe70f05a6c70","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6802","nonce":103951}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6803","nonce":103951}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6807","nonce":103951}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6806","nonce":103951}]},"public_addr":"172.21.15.12:6802/103951","cluster_addr":"172.21.15.12:6803/103951","heartbeat_back_addr":"172.21.15.12:6807/103951","heartbeat_front_addr":"172.21.15.12:6806/103951","state":["exists","up"]},{"osd":2,"uuid":"d3df7ea4-4b95-4df0-b8c0-b10295e8b5f5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.197:6801","nonce":103508}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.197:6800","nonce":103508}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.197:6804","nonce":103508}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.197:6805","nonce":103508}]},"public_addr":"172.21.15.197:6801/103508","cluster_addr":"172.21.15.197:6800/103508","heartbeat_back_addr":"172.21.15.197:6804/103508","heartbeat_front_addr":"172.21.15.197:6805/103508","state":["exists","up"]},{"osd":3,"uuid":"13666469-a3ba-4f90-8bdf-db9967e5672a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.197:6803","nonce":103512}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.197:6802","nonce":103512}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.197:6807","nonce":103512}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.197:6806","nonce":103512}]},"public_addr":"172.21.15.197:6803/103512","cluster_addr":"172.21.15.197:6802/103512","heartbeat_back_addr":"172.21.15.197:6807/103512","heartbeat_front_addr":"172.21.15.197:6806/103512","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2022-11-27T16:24:52.775 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 flush_pg_stats Ignoring malparsed backtrace: 2022-11-27T16:25:04.270 INFO:tasks.ceph.osd.1:Stopped , 2022-11-27T16:25:04.271 DEBUG:tasks.ceph.osd.2:waiting for process to exit , 2022-11-27T16:25:04.272 INFO:teuthology.orchestra.run:waiting for 300 Ignoring malparsed backtrace: 2022-11-27T16:25:32.576 INFO:teuthology.orchestra.run.smithi012.stdout:^- hv01.front.sepia.ceph.com 3 6 377 36 +87us[ +87us] +/- 121ms , 2022-11-27T16:25:32.576 INFO:teuthology.orchestra.run.smithi012.stdout:^* hv02.front.sepia.ceph.com 2 6 377 42 -241us[ -240us] +/- 46ms , 2022-11-27T16:25:32.576 INFO:teuthology.orchestra.run.smithi012.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 38 +488us[ +488us] +/- 71ms Ignoring malparsed backtrace: remote_path = remote.sh(['which', dump_program]).rstrip() , File "/home/teuthworker/src/git.ceph.com_teuthology_4da97cf64e542f347ec47b7bdbe5eca99759f9b7/teuthology/orchestra/remote.py", line 96, in sh , proc = self.run(**kwargs) Ignoring malparsed backtrace: ceph version 18.0.0-944-g03825a80 (03825a807f21359884cfd3f5cd4720b3dc4a0c78) reef (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xcc) [0x7f1958a50bee] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x1b0) [0x55b2c56bc0ca] Ignoring malparsed backtrace: 2022-11-27T16:24:52.761 INFO:teuthology.orchestra.run.smithi012.stdout: , 2022-11-27T16:24:52.761 INFO:teuthology.orchestra.run.smithi012.stdout:{"epoch":132,"fsid":"456758e0-0c99-4b3c-96eb-df14daec9506","created":"2022-11-27T15:54:12.625143+0000","modified":"2022-11-27T16:24:52.053038+0000","last_up_change":"2022-11-27T15:54:18.768155+0000","last_in_change":"2022-11-27T15:54:14.002191+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","pools":[{"pool":1,"pool_name":".mgr","create_time":"2022-11-27T15:54:20.223835+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-11-27T15:54:22.296641+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"unique_pool_0","create_time":"2022-11-27T15:54:29.448707+0000","flags":129,"flags_names":"hashpspool,write_fadvise_dontneed","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"9f1b2568-2a6e-4094-8174-af40beb33c51","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6800","nonce":103924}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6801","nonce":103924}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6804","nonce":103924}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6805","nonce":103924}]},"public_addr":"172.21.15.12:6800/103924","cluster_addr":"172.21.15.12:6801/103924","heartbeat_back_addr":"172.21.15.12:6804/103924","heartbeat_front_addr":"172.21.15.12:6805/103924","state":["exists","up"]},{"osd":1,"uuid":"7d59443e-6839-4999-93a6-fe70f05a6c70","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6802","nonce":103951}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6803","nonce":103951}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6807","nonce":103951}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6806","nonce":103951}]},"public_addr":"172.21.15.12:6802/103951","cluster_addr":"172.21.15.12:6803/103951","heartbeat_back_addr":"172.21.15.12:6807/103951","heartbeat_front_addr":"172.21.15.12:6806/103951","state":["exists","up"]},{"osd":2,"uuid":"d3df7ea4-4b95-4df0-b8c0-b10295e8b5f5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.197:6801","nonce":103508}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.197:6800","nonce":103508}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.197:6804","nonce":103508}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.197:6805","nonce":103508}]},"public_addr":"172.21.15.197:6801/103508","cluster_addr":"172.21.15.197:6800/103508","heartbeat_back_addr":"172.21.15.197:6804/103508","heartbeat_front_addr":"172.21.15.197:6805/103508","state":["exists","up"]},{"osd":3,"uuid":"13666469-a3ba-4f90-8bdf-db9967e5672a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.197:6803","nonce":103512}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.197:6802","nonce":103512}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.197:6807","nonce":103512}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.197:6806","nonce":103512}]},"public_addr":"172.21.15.197:6803/103512","cluster_addr":"172.21.15.197:6802/103512","heartbeat_back_addr":"172.21.15.197:6807/103512","heartbeat_front_addr":"172.21.15.197:6806/103512","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} , 2022-11-27T16:24:52.775 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 flush_pg_stats Ignoring malparsed backtrace: 2022-11-27T16:25:04.270 INFO:tasks.ceph.osd.1:Stopped , 2022-11-27T16:25:04.271 DEBUG:tasks.ceph.osd.2:waiting for process to exit , 2022-11-27T16:25:04.272 INFO:teuthology.orchestra.run:waiting for 300 Ignoring malparsed backtrace: 2022-11-27T16:25:32.576 INFO:teuthology.orchestra.run.smithi012.stdout:^- hv01.front.sepia.ceph.com 3 6 377 36 +87us[ +87us] +/- 121ms , 2022-11-27T16:25:32.576 INFO:teuthology.orchestra.run.smithi012.stdout:^* hv02.front.sepia.ceph.com 2 6 377 42 -241us[ -240us] +/- 46ms , 2022-11-27T16:25:32.576 INFO:teuthology.orchestra.run.smithi012.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 38 +488us[ +488us] +/- 71ms Ignoring malparsed backtrace: remote_path = remote.sh(['which', dump_program]).rstrip() , File "/home/teuthworker/src/git.ceph.com_teuthology_4da97cf64e542f347ec47b7bdbe5eca99759f9b7/teuthology/orchestra/remote.py", line 96, in sh , proc = self.run(**kwargs) Found 4 distinct failure reasons Timeout 3h running clone.client.0/qa/workunits/rbd/test_librbd.sh 1 jobs: ['7092655'] suites: ['clusters/fixed-1', 'crimson-rados/rbd/{centos_latest', 'crimson_qa_overrides', 'deploy/ceph', 'tasks/rbd_api_tests_old_format}'] Failure: Command crashed: 'CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --write-fadvise-dontneed --max-ops 4000 --objects 500 --max-in-flight 16 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 0 --op read 100 --op write 50 --op delete 10 --op write_excl 50 --pool unique_pool_0' 1 jobs: ['7092665'] suites: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{default}', 'centos_latest', 'clusters/{fixed-2}', 'crimson-rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'crimson_qa_overrides', 'deploy/ceph', 'objectstore/bluestore', 'thrashers/default', 'thrashosds-health', 'workloads/write_fadvise_dontneed}'] Failure: Command failed (workunit test rbd/test_librbd_python.sh) on smithi018 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=2d33a8c43477fc105a04fd2008dd063ba929ed14 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 CRIMSON_COMPAT=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/test_librbd_python.sh' 1 jobs: ['7092664'] suites: ['clusters/fixed-1', 'crimson-rados/rbd/{centos_latest', 'crimson_qa_overrides', 'deploy/ceph', 'tasks/rbd_python_api_tests_old_format}'] Timeout 3h running clone.client.0/qa/workunits/rbd/test_lock_fence.sh 1 jobs: ['7092661'] suites: ['clusters/fixed-1', 'crimson-rados/rbd/{centos_latest', 'crimson_qa_overrides', 'deploy/ceph', 'tasks/rbd_lock_and_fence}']