Found 445 jobs Ignoring malparsed backtrace: ceph version 17.0.0-184-g507028cb (507028cb839c25d6666833702df6325fd8569096) quincy (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f5cda87d15a] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x666) [0x55f42d6e2fe6] Ignoring malparsed backtrace: 2021-01-27T12:02:10.408 DEBUG:teuthology.orchestra.run.smithi044:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd reweight 0 1 , nodeep-scrub is unset , reweighted osd.0 to 1 (10000) Ignoring malparsed backtrace: 2021-01-27T12:02:33.295 DEBUG:teuthology.orchestra.run.smithi044:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2021-01-27T12:02:33.609 INFO:teuthology.orchestra.run.smithi044.stdout: , 2021-01-27T12:02:33.610 INFO:teuthology.orchestra.run.smithi044.stdout:{"epoch":102,"fsid":"3031f46c-60ee-43e8-9aa6-65c48a005092","created":"2021-01-27T12:00:07.307185+0000","modified":"2021-01-27T12:02:31.317484+0000","last_up_change":"2021-01-27T12:01:23.313632+0000","last_in_change":"2021-01-27T12:01:41.411093+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":7,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2021-01-27T12:00:11.876109+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2021-01-27T12:00:18.424507+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":18,"pg_placement_num":18,"pg_placement_num_target":18,"pg_num_target":18,"pg_num_pending":18,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"45","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"base","create_time":"2021-01-27T12:00:32.021550+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":4,"pg_placement_num":4,"pg_placement_num_target":4,"pg_num_target":4,"pg_num_pending":4,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"33","last_force_op_resend":"26","last_force_op_resend_prenautilus":"26","last_force_op_resend_preluminous":"26","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":24,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[4],"tier_of":-1,"read_tier":4,"write_tier":4,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}},{"pool":4,"pool_name":"cache","create_time":"2021-01-27T12:00:34.285167+0000","flags":9,"flags_names":"hashpspool,incomplete_clones","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":4,"pg_placement_num":4,"pg_placement_num_target":4,"pg_num_target":4,"pg_num_pending":4,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"33","last_force_op_resend":"26","last_force_op_resend_prenautilus":"26","last_force_op_resend_preluminous":"26","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":24,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":3,"read_tier":-1,"write_tier":-1,"cache_mode":"writeback","target_max_bytes":0,"target_max_objects":250,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"bloom","false_positive_probability":0.050000000000000003,"target_size":0,"seed":0},"hit_set_period":60,"hit_set_count":8,"use_gmt_hitset":true,"min_read_recency_for_promote":2,"min_write_recency_for_promote":2,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[1000000,1000000,1000000,1000000,1000000,1000000,1000000,1000000],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{}}],"osds":[{"osd":0,"uuid":"d8c64d60-e4e8-4924-8810-e49c7d811f8b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":73,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6808","nonce":35283}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6809","nonce":35283}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6811","nonce":35283}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6810","nonce":35283}]},"public_addr":"172.21.15.44:6808/35283","cluster_addr":"172.21.15.44:6809/35283","heartbeat_back_addr":"172.21.15.44:6811/35283","heartbeat_front_addr":"172.21.15.44:6810/35283","state":["exists","up"]},{"osd":1,"uuid":"2ac46988-25c0-44c7-ab6f-c8fd35b3a8cc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":101,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6804","nonce":35280}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6805","nonce":35280}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6807","nonce":35280}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6806","nonce":35280}]},"public_addr":"172.21.15.44:6804/35280","cluster_addr":"172.21.15.44:6805/35280","heartbeat_back_addr":"172.21.15.44:6807/35280","heartbeat_front_addr":"172.21.15.44:6806/35280","state":["exists","up"]},{"osd":2,"uuid":"c380bc4d-806c-44ae-97ba-7c06e3bc5533","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":67,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6812","nonce":35306}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6813","nonce":35306}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6815","nonce":35306}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6814","nonce":35306}]},"public_addr":"172.21.15.44:6812/35306","cluster_addr":"172.21.15.44:6813/35306","heartbeat_back_addr":"172.21.15.44:6815/35306","heartbeat_front_addr":"172.21.15.44:6814/35306","state":["exists","up"]},{"osd":3,"uuid":"2045e901-afee-463b-bd24-9dfdc7d28d44","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":74,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6800","nonce":35282}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6801","nonce":35282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6803","nonce":35282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6802","nonce":35282}]},"public_addr":"172.21.15.44:6800/35282","cluster_addr":"172.21.15.44:6801/35282","heartbeat_back_addr":"172.21.15.44:6803/35282","heartbeat_front_addr":"172.21.15.44:6802/35282","state":["exists","up"]},{"osd":4,"uuid":"b574f95c-e85a-4afb-9cbb-5a02f9f52253","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":73,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6808","nonce":35191}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6809","nonce":35191}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6811","nonce":35191}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6810","nonce":35191}]},"public_addr":"172.21.15.53:6808/35191","cluster_addr":"172.21.15.53:6809/35191","heartbeat_back_addr":"172.21.15.53:6811/35191","heartbeat_front_addr":"172.21.15.53:6810/35191","state":["exists","up"]},{"osd":5,"uuid":"09926f7f-9da0-4fee-9444-6f62c9353951","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":14,"last_clean_end":24,"up_from":54,"up_thru":63,"down_at":25,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6800","nonce":47032}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6801","nonce":47032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6803","nonce":47032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6802","nonce":47032}]},"public_addr":"172.21.15.53:6800/47032","cluster_addr":"172.21.15.53:6801/47032","heartbeat_back_addr":"172.21.15.53:6803/47032","heartbeat_front_addr":"172.21.15.53:6802/47032","state":["exists","up"]},{"osd":6,"uuid":"5b667ffb-d1e9-40d0-a306-d8b35218c09f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":72,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6804","nonce":35182}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6805","nonce":35182}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6807","nonce":35182}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6806","nonce":35182}]},"public_addr":"172.21.15.53:6804/35182","cluster_addr":"172.21.15.53:6805/35182","heartbeat_back_addr":"172.21.15.53:6807/35182","heartbeat_front_addr":"172.21.15.53:6806/35182","state":["exists","up"]},{"osd":7,"uuid":"15fa8e0f-4553-43a2-a3db-b6afd186a443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6812","nonce":35212}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6813","nonce":35212}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6815","nonce":35212}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6814","nonce":35212}]},"public_addr":"172.21.15.53:6812/35212","cluster_addr":"172.21.15.53:6813/35212","heartbeat_back_addr":"172.21.15.53:6815/35212","heartbeat_front_addr":"172.21.15.53:6814/35212","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:16.263237+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.506739+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.425556+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:14.953192+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.357603+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:24.772600+0000","dead_epoch":25},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:14.973919+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.916299+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.10","mappings":[{"from":2,"to":6},{"from":1,"to":0}]}],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} Ignoring malparsed backtrace: 2021-01-27T12:03:00.378 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2021-01-27T12:02:45.880587+0000 time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=45, tm_wday=2, tm_yday=27, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=27, tm_isdst=0) , 2021-01-27T12:03:00.378 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2021-01-27T12:02:36.128894+0000 time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=36, tm_wday=2, tm_yday=27, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=27, tm_isdst=0) , 2021-01-27T12:03:00.379 INFO:tasks.ceph:pgid 4.0 last_scrub_stamp 2021-01-27T12:01:52.526935+0000 time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=1, tm_sec=52, tm_wday=2, tm_yday=27, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=27, tm_isdst=0) Ignoring malparsed backtrace: 2021-01-27T12:03:42.890 DEBUG:teuthology.orchestra.run.smithi053:> sync && sudo umount -f /var/lib/ceph/osd/ceph-5 , 2021-01-27T12:03:42.978 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-6 on ubuntu@smithi053.front.sepia.ceph.com , 2021-01-27T12:03:42.978 DEBUG:teuthology.orchestra.run.smithi053:> sync && sudo umount -f /var/lib/ceph/osd/ceph-6 Ignoring malparsed backtrace: 2021-01-27T12:04:00.247 DEBUG:teuthology.orchestra.run.smithi044:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log , 2021-01-27T12:04:00.325 DEBUG:teuthology.orchestra.run.smithi053:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log , 2021-01-27T12:04:00.389 DEBUG:teuthology.orchestra.run.smithi044:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av '\(comm="dmidecode"\|chronyd.service\|name="cephtest"\|scontext=system_u:system_r:nrpe_t:s0\|scontext=system_u:system_r:pcp_pmlogger_t\|scontext=system_u:system_r:pcp_pmcd_t:s0\|comm="rhsmd"\|scontext=system_u:system_r:syslogd_t:s0\|tcontext=system_u:system_r:nrpe_t:s0\|comm="updatedb"\|comm="smartd"\|comm="rhsmcertd-worke"\|comm="setroubleshootd"\|comm="rpm"\|tcontext=system_u:object_r:container_runtime_exec_t:s0\|comm="ksmtuned"\|scontext=system_u:system_r:logrotate_t:s0\)' Ignoring malparsed backtrace: ceph version 17.0.0-184-g507028cb (507028cb839c25d6666833702df6325fd8569096) quincy (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f5cda87d15a] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x666) [0x55f42d6e2fe6] Ignoring malparsed backtrace: 2021-01-27T12:02:10.408 DEBUG:teuthology.orchestra.run.smithi044:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd reweight 0 1 , nodeep-scrub is unset , reweighted osd.0 to 1 (10000) Ignoring malparsed backtrace: 2021-01-27T12:02:33.295 DEBUG:teuthology.orchestra.run.smithi044:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2021-01-27T12:02:33.609 INFO:teuthology.orchestra.run.smithi044.stdout: , 2021-01-27T12:02:33.610 INFO:teuthology.orchestra.run.smithi044.stdout:{"epoch":102,"fsid":"3031f46c-60ee-43e8-9aa6-65c48a005092","created":"2021-01-27T12:00:07.307185+0000","modified":"2021-01-27T12:02:31.317484+0000","last_up_change":"2021-01-27T12:01:23.313632+0000","last_in_change":"2021-01-27T12:01:41.411093+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":7,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2021-01-27T12:00:11.876109+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2021-01-27T12:00:18.424507+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":18,"pg_placement_num":18,"pg_placement_num_target":18,"pg_num_target":18,"pg_num_pending":18,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"45","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"base","create_time":"2021-01-27T12:00:32.021550+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":4,"pg_placement_num":4,"pg_placement_num_target":4,"pg_num_target":4,"pg_num_pending":4,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"33","last_force_op_resend":"26","last_force_op_resend_prenautilus":"26","last_force_op_resend_preluminous":"26","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":24,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[4],"tier_of":-1,"read_tier":4,"write_tier":4,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}},{"pool":4,"pool_name":"cache","create_time":"2021-01-27T12:00:34.285167+0000","flags":9,"flags_names":"hashpspool,incomplete_clones","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":4,"pg_placement_num":4,"pg_placement_num_target":4,"pg_num_target":4,"pg_num_pending":4,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"33","last_force_op_resend":"26","last_force_op_resend_prenautilus":"26","last_force_op_resend_preluminous":"26","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":24,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":3,"read_tier":-1,"write_tier":-1,"cache_mode":"writeback","target_max_bytes":0,"target_max_objects":250,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"bloom","false_positive_probability":0.050000000000000003,"target_size":0,"seed":0},"hit_set_period":60,"hit_set_count":8,"use_gmt_hitset":true,"min_read_recency_for_promote":2,"min_write_recency_for_promote":2,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[1000000,1000000,1000000,1000000,1000000,1000000,1000000,1000000],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{}}],"osds":[{"osd":0,"uuid":"d8c64d60-e4e8-4924-8810-e49c7d811f8b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":73,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6808","nonce":35283}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6809","nonce":35283}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6811","nonce":35283}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6810","nonce":35283}]},"public_addr":"172.21.15.44:6808/35283","cluster_addr":"172.21.15.44:6809/35283","heartbeat_back_addr":"172.21.15.44:6811/35283","heartbeat_front_addr":"172.21.15.44:6810/35283","state":["exists","up"]},{"osd":1,"uuid":"2ac46988-25c0-44c7-ab6f-c8fd35b3a8cc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":101,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6804","nonce":35280}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6805","nonce":35280}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6807","nonce":35280}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6806","nonce":35280}]},"public_addr":"172.21.15.44:6804/35280","cluster_addr":"172.21.15.44:6805/35280","heartbeat_back_addr":"172.21.15.44:6807/35280","heartbeat_front_addr":"172.21.15.44:6806/35280","state":["exists","up"]},{"osd":2,"uuid":"c380bc4d-806c-44ae-97ba-7c06e3bc5533","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":67,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6812","nonce":35306}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6813","nonce":35306}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6815","nonce":35306}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6814","nonce":35306}]},"public_addr":"172.21.15.44:6812/35306","cluster_addr":"172.21.15.44:6813/35306","heartbeat_back_addr":"172.21.15.44:6815/35306","heartbeat_front_addr":"172.21.15.44:6814/35306","state":["exists","up"]},{"osd":3,"uuid":"2045e901-afee-463b-bd24-9dfdc7d28d44","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":74,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6800","nonce":35282}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6801","nonce":35282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6803","nonce":35282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6802","nonce":35282}]},"public_addr":"172.21.15.44:6800/35282","cluster_addr":"172.21.15.44:6801/35282","heartbeat_back_addr":"172.21.15.44:6803/35282","heartbeat_front_addr":"172.21.15.44:6802/35282","state":["exists","up"]},{"osd":4,"uuid":"b574f95c-e85a-4afb-9cbb-5a02f9f52253","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":73,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6808","nonce":35191}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6809","nonce":35191}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6811","nonce":35191}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6810","nonce":35191}]},"public_addr":"172.21.15.53:6808/35191","cluster_addr":"172.21.15.53:6809/35191","heartbeat_back_addr":"172.21.15.53:6811/35191","heartbeat_front_addr":"172.21.15.53:6810/35191","state":["exists","up"]},{"osd":5,"uuid":"09926f7f-9da0-4fee-9444-6f62c9353951","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":14,"last_clean_end":24,"up_from":54,"up_thru":63,"down_at":25,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6800","nonce":47032}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6801","nonce":47032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6803","nonce":47032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6802","nonce":47032}]},"public_addr":"172.21.15.53:6800/47032","cluster_addr":"172.21.15.53:6801/47032","heartbeat_back_addr":"172.21.15.53:6803/47032","heartbeat_front_addr":"172.21.15.53:6802/47032","state":["exists","up"]},{"osd":6,"uuid":"5b667ffb-d1e9-40d0-a306-d8b35218c09f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":72,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6804","nonce":35182}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6805","nonce":35182}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6807","nonce":35182}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6806","nonce":35182}]},"public_addr":"172.21.15.53:6804/35182","cluster_addr":"172.21.15.53:6805/35182","heartbeat_back_addr":"172.21.15.53:6807/35182","heartbeat_front_addr":"172.21.15.53:6806/35182","state":["exists","up"]},{"osd":7,"uuid":"15fa8e0f-4553-43a2-a3db-b6afd186a443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6812","nonce":35212}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6813","nonce":35212}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6815","nonce":35212}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6814","nonce":35212}]},"public_addr":"172.21.15.53:6812/35212","cluster_addr":"172.21.15.53:6813/35212","heartbeat_back_addr":"172.21.15.53:6815/35212","heartbeat_front_addr":"172.21.15.53:6814/35212","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:16.263237+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.506739+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.425556+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:14.953192+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.357603+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:24.772600+0000","dead_epoch":25},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:14.973919+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.916299+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.10","mappings":[{"from":2,"to":6},{"from":1,"to":0}]}],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} Ignoring malparsed backtrace: 2021-01-27T12:03:00.378 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2021-01-27T12:02:45.880587+0000 time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=45, tm_wday=2, tm_yday=27, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=27, tm_isdst=0) , 2021-01-27T12:03:00.378 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2021-01-27T12:02:36.128894+0000 time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=36, tm_wday=2, tm_yday=27, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=27, tm_isdst=0) , 2021-01-27T12:03:00.379 INFO:tasks.ceph:pgid 4.0 last_scrub_stamp 2021-01-27T12:01:52.526935+0000 time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=1, tm_sec=52, tm_wday=2, tm_yday=27, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=27, tm_isdst=0) Ignoring malparsed backtrace: 2021-01-27T12:03:42.890 DEBUG:teuthology.orchestra.run.smithi053:> sync && sudo umount -f /var/lib/ceph/osd/ceph-5 , 2021-01-27T12:03:42.978 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-6 on ubuntu@smithi053.front.sepia.ceph.com , 2021-01-27T12:03:42.978 DEBUG:teuthology.orchestra.run.smithi053:> sync && sudo umount -f /var/lib/ceph/osd/ceph-6 Ignoring malparsed backtrace: 2021-01-27T12:04:00.247 DEBUG:teuthology.orchestra.run.smithi044:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log , 2021-01-27T12:04:00.325 DEBUG:teuthology.orchestra.run.smithi053:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log , 2021-01-27T12:04:00.389 DEBUG:teuthology.orchestra.run.smithi044:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av '\(comm="dmidecode"\|chronyd.service\|name="cephtest"\|scontext=system_u:system_r:nrpe_t:s0\|scontext=system_u:system_r:pcp_pmlogger_t\|scontext=system_u:system_r:pcp_pmcd_t:s0\|comm="rhsmd"\|scontext=system_u:system_r:syslogd_t:s0\|tcontext=system_u:system_r:nrpe_t:s0\|comm="updatedb"\|comm="smartd"\|comm="rhsmcertd-worke"\|comm="setroubleshootd"\|comm="rpm"\|tcontext=system_u:object_r:container_runtime_exec_t:s0\|comm="ksmtuned"\|scontext=system_u:system_r:logrotate_t:s0\)' Ignoring malparsed backtrace: ceph version 17.0.0-184-g507028cb (507028cb839c25d6666833702df6325fd8569096) quincy (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f5cda87d15a] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x666) [0x55f42d6e2fe6] Ignoring malparsed backtrace: 2021-01-27T12:02:10.408 DEBUG:teuthology.orchestra.run.smithi044:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd reweight 0 1 , nodeep-scrub is unset , reweighted osd.0 to 1 (10000) Ignoring malparsed backtrace: 2021-01-27T12:02:33.295 DEBUG:teuthology.orchestra.run.smithi044:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2021-01-27T12:02:33.609 INFO:teuthology.orchestra.run.smithi044.stdout: , 2021-01-27T12:02:33.610 INFO:teuthology.orchestra.run.smithi044.stdout:{"epoch":102,"fsid":"3031f46c-60ee-43e8-9aa6-65c48a005092","created":"2021-01-27T12:00:07.307185+0000","modified":"2021-01-27T12:02:31.317484+0000","last_up_change":"2021-01-27T12:01:23.313632+0000","last_in_change":"2021-01-27T12:01:41.411093+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":7,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2021-01-27T12:00:11.876109+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2021-01-27T12:00:18.424507+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":18,"pg_placement_num":18,"pg_placement_num_target":18,"pg_num_target":18,"pg_num_pending":18,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"45","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"base","create_time":"2021-01-27T12:00:32.021550+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":4,"pg_placement_num":4,"pg_placement_num_target":4,"pg_num_target":4,"pg_num_pending":4,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"33","last_force_op_resend":"26","last_force_op_resend_prenautilus":"26","last_force_op_resend_preluminous":"26","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":24,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[4],"tier_of":-1,"read_tier":4,"write_tier":4,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}},{"pool":4,"pool_name":"cache","create_time":"2021-01-27T12:00:34.285167+0000","flags":9,"flags_names":"hashpspool,incomplete_clones","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":4,"pg_placement_num":4,"pg_placement_num_target":4,"pg_num_target":4,"pg_num_pending":4,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"33","last_force_op_resend":"26","last_force_op_resend_prenautilus":"26","last_force_op_resend_preluminous":"26","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":24,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":3,"read_tier":-1,"write_tier":-1,"cache_mode":"writeback","target_max_bytes":0,"target_max_objects":250,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"bloom","false_positive_probability":0.050000000000000003,"target_size":0,"seed":0},"hit_set_period":60,"hit_set_count":8,"use_gmt_hitset":true,"min_read_recency_for_promote":2,"min_write_recency_for_promote":2,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[1000000,1000000,1000000,1000000,1000000,1000000,1000000,1000000],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{}}],"osds":[{"osd":0,"uuid":"d8c64d60-e4e8-4924-8810-e49c7d811f8b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":73,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6808","nonce":35283}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6809","nonce":35283}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6811","nonce":35283}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6810","nonce":35283}]},"public_addr":"172.21.15.44:6808/35283","cluster_addr":"172.21.15.44:6809/35283","heartbeat_back_addr":"172.21.15.44:6811/35283","heartbeat_front_addr":"172.21.15.44:6810/35283","state":["exists","up"]},{"osd":1,"uuid":"2ac46988-25c0-44c7-ab6f-c8fd35b3a8cc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":101,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6804","nonce":35280}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6805","nonce":35280}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6807","nonce":35280}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6806","nonce":35280}]},"public_addr":"172.21.15.44:6804/35280","cluster_addr":"172.21.15.44:6805/35280","heartbeat_back_addr":"172.21.15.44:6807/35280","heartbeat_front_addr":"172.21.15.44:6806/35280","state":["exists","up"]},{"osd":2,"uuid":"c380bc4d-806c-44ae-97ba-7c06e3bc5533","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":67,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6812","nonce":35306}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6813","nonce":35306}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6815","nonce":35306}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6814","nonce":35306}]},"public_addr":"172.21.15.44:6812/35306","cluster_addr":"172.21.15.44:6813/35306","heartbeat_back_addr":"172.21.15.44:6815/35306","heartbeat_front_addr":"172.21.15.44:6814/35306","state":["exists","up"]},{"osd":3,"uuid":"2045e901-afee-463b-bd24-9dfdc7d28d44","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":74,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6800","nonce":35282}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6801","nonce":35282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6803","nonce":35282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6802","nonce":35282}]},"public_addr":"172.21.15.44:6800/35282","cluster_addr":"172.21.15.44:6801/35282","heartbeat_back_addr":"172.21.15.44:6803/35282","heartbeat_front_addr":"172.21.15.44:6802/35282","state":["exists","up"]},{"osd":4,"uuid":"b574f95c-e85a-4afb-9cbb-5a02f9f52253","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":73,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6808","nonce":35191}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6809","nonce":35191}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6811","nonce":35191}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6810","nonce":35191}]},"public_addr":"172.21.15.53:6808/35191","cluster_addr":"172.21.15.53:6809/35191","heartbeat_back_addr":"172.21.15.53:6811/35191","heartbeat_front_addr":"172.21.15.53:6810/35191","state":["exists","up"]},{"osd":5,"uuid":"09926f7f-9da0-4fee-9444-6f62c9353951","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":14,"last_clean_end":24,"up_from":54,"up_thru":63,"down_at":25,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6800","nonce":47032}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6801","nonce":47032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6803","nonce":47032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6802","nonce":47032}]},"public_addr":"172.21.15.53:6800/47032","cluster_addr":"172.21.15.53:6801/47032","heartbeat_back_addr":"172.21.15.53:6803/47032","heartbeat_front_addr":"172.21.15.53:6802/47032","state":["exists","up"]},{"osd":6,"uuid":"5b667ffb-d1e9-40d0-a306-d8b35218c09f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":72,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6804","nonce":35182}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6805","nonce":35182}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6807","nonce":35182}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6806","nonce":35182}]},"public_addr":"172.21.15.53:6804/35182","cluster_addr":"172.21.15.53:6805/35182","heartbeat_back_addr":"172.21.15.53:6807/35182","heartbeat_front_addr":"172.21.15.53:6806/35182","state":["exists","up"]},{"osd":7,"uuid":"15fa8e0f-4553-43a2-a3db-b6afd186a443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6812","nonce":35212}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6813","nonce":35212}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6815","nonce":35212}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6814","nonce":35212}]},"public_addr":"172.21.15.53:6812/35212","cluster_addr":"172.21.15.53:6813/35212","heartbeat_back_addr":"172.21.15.53:6815/35212","heartbeat_front_addr":"172.21.15.53:6814/35212","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:16.263237+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.506739+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.425556+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:14.953192+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.357603+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:24.772600+0000","dead_epoch":25},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:14.973919+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.916299+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.10","mappings":[{"from":2,"to":6},{"from":1,"to":0}]}],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} Ignoring malparsed backtrace: 2021-01-27T12:03:00.378 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2021-01-27T12:02:45.880587+0000 time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=45, tm_wday=2, tm_yday=27, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=27, tm_isdst=0) , 2021-01-27T12:03:00.378 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2021-01-27T12:02:36.128894+0000 time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=36, tm_wday=2, tm_yday=27, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=27, tm_isdst=0) , 2021-01-27T12:03:00.379 INFO:tasks.ceph:pgid 4.0 last_scrub_stamp 2021-01-27T12:01:52.526935+0000 time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=1, tm_sec=52, tm_wday=2, tm_yday=27, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=27, tm_isdst=0) Ignoring malparsed backtrace: 2021-01-27T12:03:42.890 DEBUG:teuthology.orchestra.run.smithi053:> sync && sudo umount -f /var/lib/ceph/osd/ceph-5 , 2021-01-27T12:03:42.978 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-6 on ubuntu@smithi053.front.sepia.ceph.com , 2021-01-27T12:03:42.978 DEBUG:teuthology.orchestra.run.smithi053:> sync && sudo umount -f /var/lib/ceph/osd/ceph-6 Ignoring malparsed backtrace: 2021-01-27T12:04:00.247 DEBUG:teuthology.orchestra.run.smithi044:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log , 2021-01-27T12:04:00.325 DEBUG:teuthology.orchestra.run.smithi053:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log , 2021-01-27T12:04:00.389 DEBUG:teuthology.orchestra.run.smithi044:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av '\(comm="dmidecode"\|chronyd.service\|name="cephtest"\|scontext=system_u:system_r:nrpe_t:s0\|scontext=system_u:system_r:pcp_pmlogger_t\|scontext=system_u:system_r:pcp_pmcd_t:s0\|comm="rhsmd"\|scontext=system_u:system_r:syslogd_t:s0\|tcontext=system_u:system_r:nrpe_t:s0\|comm="updatedb"\|comm="smartd"\|comm="rhsmcertd-worke"\|comm="setroubleshootd"\|comm="rpm"\|tcontext=system_u:object_r:container_runtime_exec_t:s0\|comm="ksmtuned"\|scontext=system_u:system_r:logrotate_t:s0\)' Ignoring malparsed backtrace: ceph version 17.0.0-184-g507028cb (507028cb839c25d6666833702df6325fd8569096) quincy (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f5cda87d15a] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x666) [0x55f42d6e2fe6] Ignoring malparsed backtrace: 2021-01-27T12:02:10.408 DEBUG:teuthology.orchestra.run.smithi044:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd reweight 0 1 , nodeep-scrub is unset , reweighted osd.0 to 1 (10000) Ignoring malparsed backtrace: 2021-01-27T12:02:33.295 DEBUG:teuthology.orchestra.run.smithi044:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2021-01-27T12:02:33.609 INFO:teuthology.orchestra.run.smithi044.stdout: , 2021-01-27T12:02:33.610 INFO:teuthology.orchestra.run.smithi044.stdout:{"epoch":102,"fsid":"3031f46c-60ee-43e8-9aa6-65c48a005092","created":"2021-01-27T12:00:07.307185+0000","modified":"2021-01-27T12:02:31.317484+0000","last_up_change":"2021-01-27T12:01:23.313632+0000","last_in_change":"2021-01-27T12:01:41.411093+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":7,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2021-01-27T12:00:11.876109+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2021-01-27T12:00:18.424507+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":18,"pg_placement_num":18,"pg_placement_num_target":18,"pg_num_target":18,"pg_num_pending":18,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"45","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"base","create_time":"2021-01-27T12:00:32.021550+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":4,"pg_placement_num":4,"pg_placement_num_target":4,"pg_num_target":4,"pg_num_pending":4,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"33","last_force_op_resend":"26","last_force_op_resend_prenautilus":"26","last_force_op_resend_preluminous":"26","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":24,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[4],"tier_of":-1,"read_tier":4,"write_tier":4,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}},{"pool":4,"pool_name":"cache","create_time":"2021-01-27T12:00:34.285167+0000","flags":9,"flags_names":"hashpspool,incomplete_clones","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":4,"pg_placement_num":4,"pg_placement_num_target":4,"pg_num_target":4,"pg_num_pending":4,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"33","last_force_op_resend":"26","last_force_op_resend_prenautilus":"26","last_force_op_resend_preluminous":"26","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":24,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":3,"read_tier":-1,"write_tier":-1,"cache_mode":"writeback","target_max_bytes":0,"target_max_objects":250,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"bloom","false_positive_probability":0.050000000000000003,"target_size":0,"seed":0},"hit_set_period":60,"hit_set_count":8,"use_gmt_hitset":true,"min_read_recency_for_promote":2,"min_write_recency_for_promote":2,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[1000000,1000000,1000000,1000000,1000000,1000000,1000000,1000000],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{}}],"osds":[{"osd":0,"uuid":"d8c64d60-e4e8-4924-8810-e49c7d811f8b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":73,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6808","nonce":35283}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6809","nonce":35283}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6811","nonce":35283}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6810","nonce":35283}]},"public_addr":"172.21.15.44:6808/35283","cluster_addr":"172.21.15.44:6809/35283","heartbeat_back_addr":"172.21.15.44:6811/35283","heartbeat_front_addr":"172.21.15.44:6810/35283","state":["exists","up"]},{"osd":1,"uuid":"2ac46988-25c0-44c7-ab6f-c8fd35b3a8cc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":101,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6804","nonce":35280}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6805","nonce":35280}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6807","nonce":35280}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6806","nonce":35280}]},"public_addr":"172.21.15.44:6804/35280","cluster_addr":"172.21.15.44:6805/35280","heartbeat_back_addr":"172.21.15.44:6807/35280","heartbeat_front_addr":"172.21.15.44:6806/35280","state":["exists","up"]},{"osd":2,"uuid":"c380bc4d-806c-44ae-97ba-7c06e3bc5533","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":67,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6812","nonce":35306}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6813","nonce":35306}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6815","nonce":35306}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6814","nonce":35306}]},"public_addr":"172.21.15.44:6812/35306","cluster_addr":"172.21.15.44:6813/35306","heartbeat_back_addr":"172.21.15.44:6815/35306","heartbeat_front_addr":"172.21.15.44:6814/35306","state":["exists","up"]},{"osd":3,"uuid":"2045e901-afee-463b-bd24-9dfdc7d28d44","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":74,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6800","nonce":35282}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6801","nonce":35282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6803","nonce":35282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6802","nonce":35282}]},"public_addr":"172.21.15.44:6800/35282","cluster_addr":"172.21.15.44:6801/35282","heartbeat_back_addr":"172.21.15.44:6803/35282","heartbeat_front_addr":"172.21.15.44:6802/35282","state":["exists","up"]},{"osd":4,"uuid":"b574f95c-e85a-4afb-9cbb-5a02f9f52253","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":73,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6808","nonce":35191}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6809","nonce":35191}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6811","nonce":35191}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6810","nonce":35191}]},"public_addr":"172.21.15.53:6808/35191","cluster_addr":"172.21.15.53:6809/35191","heartbeat_back_addr":"172.21.15.53:6811/35191","heartbeat_front_addr":"172.21.15.53:6810/35191","state":["exists","up"]},{"osd":5,"uuid":"09926f7f-9da0-4fee-9444-6f62c9353951","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":14,"last_clean_end":24,"up_from":54,"up_thru":63,"down_at":25,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6800","nonce":47032}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6801","nonce":47032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6803","nonce":47032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6802","nonce":47032}]},"public_addr":"172.21.15.53:6800/47032","cluster_addr":"172.21.15.53:6801/47032","heartbeat_back_addr":"172.21.15.53:6803/47032","heartbeat_front_addr":"172.21.15.53:6802/47032","state":["exists","up"]},{"osd":6,"uuid":"5b667ffb-d1e9-40d0-a306-d8b35218c09f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":72,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6804","nonce":35182}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6805","nonce":35182}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6807","nonce":35182}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6806","nonce":35182}]},"public_addr":"172.21.15.53:6804/35182","cluster_addr":"172.21.15.53:6805/35182","heartbeat_back_addr":"172.21.15.53:6807/35182","heartbeat_front_addr":"172.21.15.53:6806/35182","state":["exists","up"]},{"osd":7,"uuid":"15fa8e0f-4553-43a2-a3db-b6afd186a443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6812","nonce":35212}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6813","nonce":35212}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6815","nonce":35212}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6814","nonce":35212}]},"public_addr":"172.21.15.53:6812/35212","cluster_addr":"172.21.15.53:6813/35212","heartbeat_back_addr":"172.21.15.53:6815/35212","heartbeat_front_addr":"172.21.15.53:6814/35212","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:16.263237+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.506739+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.425556+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:14.953192+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.357603+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:24.772600+0000","dead_epoch":25},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:14.973919+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.916299+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.10","mappings":[{"from":2,"to":6},{"from":1,"to":0}]}],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} Ignoring malparsed backtrace: 2021-01-27T12:03:00.378 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2021-01-27T12:02:45.880587+0000 time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=45, tm_wday=2, tm_yday=27, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=27, tm_isdst=0) , 2021-01-27T12:03:00.378 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2021-01-27T12:02:36.128894+0000 time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=36, tm_wday=2, tm_yday=27, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=27, tm_isdst=0) , 2021-01-27T12:03:00.379 INFO:tasks.ceph:pgid 4.0 last_scrub_stamp 2021-01-27T12:01:52.526935+0000 time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=1, tm_sec=52, tm_wday=2, tm_yday=27, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=27, tm_isdst=0) Ignoring malparsed backtrace: 2021-01-27T12:03:42.890 DEBUG:teuthology.orchestra.run.smithi053:> sync && sudo umount -f /var/lib/ceph/osd/ceph-5 , 2021-01-27T12:03:42.978 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-6 on ubuntu@smithi053.front.sepia.ceph.com , 2021-01-27T12:03:42.978 DEBUG:teuthology.orchestra.run.smithi053:> sync && sudo umount -f /var/lib/ceph/osd/ceph-6 Ignoring malparsed backtrace: 2021-01-27T12:04:00.247 DEBUG:teuthology.orchestra.run.smithi044:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log , 2021-01-27T12:04:00.325 DEBUG:teuthology.orchestra.run.smithi053:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log , 2021-01-27T12:04:00.389 DEBUG:teuthology.orchestra.run.smithi044:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av '\(comm="dmidecode"\|chronyd.service\|name="cephtest"\|scontext=system_u:system_r:nrpe_t:s0\|scontext=system_u:system_r:pcp_pmlogger_t\|scontext=system_u:system_r:pcp_pmcd_t:s0\|comm="rhsmd"\|scontext=system_u:system_r:syslogd_t:s0\|tcontext=system_u:system_r:nrpe_t:s0\|comm="updatedb"\|comm="smartd"\|comm="rhsmcertd-worke"\|comm="setroubleshootd"\|comm="rpm"\|tcontext=system_u:object_r:container_runtime_exec_t:s0\|comm="ksmtuned"\|scontext=system_u:system_r:logrotate_t:s0\)' Ignoring malparsed backtrace: ceph version 17.0.0-184-g507028cb (507028cb839c25d6666833702df6325fd8569096) quincy (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f5cda87d15a] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x666) [0x55f42d6e2fe6] Ignoring malparsed backtrace: 2021-01-27T12:02:10.408 DEBUG:teuthology.orchestra.run.smithi044:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd reweight 0 1 , nodeep-scrub is unset , reweighted osd.0 to 1 (10000) Ignoring malparsed backtrace: 2021-01-27T12:02:33.295 DEBUG:teuthology.orchestra.run.smithi044:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2021-01-27T12:02:33.609 INFO:teuthology.orchestra.run.smithi044.stdout: , 2021-01-27T12:02:33.610 INFO:teuthology.orchestra.run.smithi044.stdout:{"epoch":102,"fsid":"3031f46c-60ee-43e8-9aa6-65c48a005092","created":"2021-01-27T12:00:07.307185+0000","modified":"2021-01-27T12:02:31.317484+0000","last_up_change":"2021-01-27T12:01:23.313632+0000","last_in_change":"2021-01-27T12:01:41.411093+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":7,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2021-01-27T12:00:11.876109+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2021-01-27T12:00:18.424507+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":18,"pg_placement_num":18,"pg_placement_num_target":18,"pg_num_target":18,"pg_num_pending":18,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"45","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"base","create_time":"2021-01-27T12:00:32.021550+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":4,"pg_placement_num":4,"pg_placement_num_target":4,"pg_num_target":4,"pg_num_pending":4,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"33","last_force_op_resend":"26","last_force_op_resend_prenautilus":"26","last_force_op_resend_preluminous":"26","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":24,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[4],"tier_of":-1,"read_tier":4,"write_tier":4,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}},{"pool":4,"pool_name":"cache","create_time":"2021-01-27T12:00:34.285167+0000","flags":9,"flags_names":"hashpspool,incomplete_clones","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":4,"pg_placement_num":4,"pg_placement_num_target":4,"pg_num_target":4,"pg_num_pending":4,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"33","last_force_op_resend":"26","last_force_op_resend_prenautilus":"26","last_force_op_resend_preluminous":"26","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":24,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":3,"read_tier":-1,"write_tier":-1,"cache_mode":"writeback","target_max_bytes":0,"target_max_objects":250,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"bloom","false_positive_probability":0.050000000000000003,"target_size":0,"seed":0},"hit_set_period":60,"hit_set_count":8,"use_gmt_hitset":true,"min_read_recency_for_promote":2,"min_write_recency_for_promote":2,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[1000000,1000000,1000000,1000000,1000000,1000000,1000000,1000000],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{}}],"osds":[{"osd":0,"uuid":"d8c64d60-e4e8-4924-8810-e49c7d811f8b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":73,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6808","nonce":35283}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6809","nonce":35283}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6811","nonce":35283}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6810","nonce":35283}]},"public_addr":"172.21.15.44:6808/35283","cluster_addr":"172.21.15.44:6809/35283","heartbeat_back_addr":"172.21.15.44:6811/35283","heartbeat_front_addr":"172.21.15.44:6810/35283","state":["exists","up"]},{"osd":1,"uuid":"2ac46988-25c0-44c7-ab6f-c8fd35b3a8cc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":101,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6804","nonce":35280}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6805","nonce":35280}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6807","nonce":35280}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6806","nonce":35280}]},"public_addr":"172.21.15.44:6804/35280","cluster_addr":"172.21.15.44:6805/35280","heartbeat_back_addr":"172.21.15.44:6807/35280","heartbeat_front_addr":"172.21.15.44:6806/35280","state":["exists","up"]},{"osd":2,"uuid":"c380bc4d-806c-44ae-97ba-7c06e3bc5533","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":67,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6812","nonce":35306}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6813","nonce":35306}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6815","nonce":35306}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6814","nonce":35306}]},"public_addr":"172.21.15.44:6812/35306","cluster_addr":"172.21.15.44:6813/35306","heartbeat_back_addr":"172.21.15.44:6815/35306","heartbeat_front_addr":"172.21.15.44:6814/35306","state":["exists","up"]},{"osd":3,"uuid":"2045e901-afee-463b-bd24-9dfdc7d28d44","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":74,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6800","nonce":35282}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6801","nonce":35282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6803","nonce":35282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6802","nonce":35282}]},"public_addr":"172.21.15.44:6800/35282","cluster_addr":"172.21.15.44:6801/35282","heartbeat_back_addr":"172.21.15.44:6803/35282","heartbeat_front_addr":"172.21.15.44:6802/35282","state":["exists","up"]},{"osd":4,"uuid":"b574f95c-e85a-4afb-9cbb-5a02f9f52253","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":73,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6808","nonce":35191}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6809","nonce":35191}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6811","nonce":35191}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6810","nonce":35191}]},"public_addr":"172.21.15.53:6808/35191","cluster_addr":"172.21.15.53:6809/35191","heartbeat_back_addr":"172.21.15.53:6811/35191","heartbeat_front_addr":"172.21.15.53:6810/35191","state":["exists","up"]},{"osd":5,"uuid":"09926f7f-9da0-4fee-9444-6f62c9353951","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":14,"last_clean_end":24,"up_from":54,"up_thru":63,"down_at":25,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6800","nonce":47032}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6801","nonce":47032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6803","nonce":47032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6802","nonce":47032}]},"public_addr":"172.21.15.53:6800/47032","cluster_addr":"172.21.15.53:6801/47032","heartbeat_back_addr":"172.21.15.53:6803/47032","heartbeat_front_addr":"172.21.15.53:6802/47032","state":["exists","up"]},{"osd":6,"uuid":"5b667ffb-d1e9-40d0-a306-d8b35218c09f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":72,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6804","nonce":35182}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6805","nonce":35182}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6807","nonce":35182}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6806","nonce":35182}]},"public_addr":"172.21.15.53:6804/35182","cluster_addr":"172.21.15.53:6805/35182","heartbeat_back_addr":"172.21.15.53:6807/35182","heartbeat_front_addr":"172.21.15.53:6806/35182","state":["exists","up"]},{"osd":7,"uuid":"15fa8e0f-4553-43a2-a3db-b6afd186a443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6812","nonce":35212}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6813","nonce":35212}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6815","nonce":35212}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6814","nonce":35212}]},"public_addr":"172.21.15.53:6812/35212","cluster_addr":"172.21.15.53:6813/35212","heartbeat_back_addr":"172.21.15.53:6815/35212","heartbeat_front_addr":"172.21.15.53:6814/35212","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:16.263237+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.506739+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.425556+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:14.953192+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.357603+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:24.772600+0000","dead_epoch":25},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:14.973919+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.916299+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.10","mappings":[{"from":2,"to":6},{"from":1,"to":0}]}],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} Ignoring malparsed backtrace: 2021-01-27T12:03:00.378 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2021-01-27T12:02:45.880587+0000 time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=45, tm_wday=2, tm_yday=27, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=27, tm_isdst=0) , 2021-01-27T12:03:00.378 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2021-01-27T12:02:36.128894+0000 time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=36, tm_wday=2, tm_yday=27, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=27, tm_isdst=0) , 2021-01-27T12:03:00.379 INFO:tasks.ceph:pgid 4.0 last_scrub_stamp 2021-01-27T12:01:52.526935+0000 time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=1, tm_sec=52, tm_wday=2, tm_yday=27, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=27, tm_isdst=0) Ignoring malparsed backtrace: 2021-01-27T12:03:42.890 DEBUG:teuthology.orchestra.run.smithi053:> sync && sudo umount -f /var/lib/ceph/osd/ceph-5 , 2021-01-27T12:03:42.978 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-6 on ubuntu@smithi053.front.sepia.ceph.com , 2021-01-27T12:03:42.978 DEBUG:teuthology.orchestra.run.smithi053:> sync && sudo umount -f /var/lib/ceph/osd/ceph-6 Ignoring malparsed backtrace: 2021-01-27T12:04:00.247 DEBUG:teuthology.orchestra.run.smithi044:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log , 2021-01-27T12:04:00.325 DEBUG:teuthology.orchestra.run.smithi053:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log , 2021-01-27T12:04:00.389 DEBUG:teuthology.orchestra.run.smithi044:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av '\(comm="dmidecode"\|chronyd.service\|name="cephtest"\|scontext=system_u:system_r:nrpe_t:s0\|scontext=system_u:system_r:pcp_pmlogger_t\|scontext=system_u:system_r:pcp_pmcd_t:s0\|comm="rhsmd"\|scontext=system_u:system_r:syslogd_t:s0\|tcontext=system_u:system_r:nrpe_t:s0\|comm="updatedb"\|comm="smartd"\|comm="rhsmcertd-worke"\|comm="setroubleshootd"\|comm="rpm"\|tcontext=system_u:object_r:container_runtime_exec_t:s0\|comm="ksmtuned"\|scontext=system_u:system_r:logrotate_t:s0\)' Ignoring malparsed backtrace: ceph version 17.0.0-184-g507028cb (507028cb839c25d6666833702df6325fd8569096) quincy (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f5cda87d15a] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x666) [0x55f42d6e2fe6] Ignoring malparsed backtrace: 2021-01-27T12:02:10.408 DEBUG:teuthology.orchestra.run.smithi044:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd reweight 0 1 , nodeep-scrub is unset , reweighted osd.0 to 1 (10000) Ignoring malparsed backtrace: 2021-01-27T12:02:33.295 DEBUG:teuthology.orchestra.run.smithi044:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2021-01-27T12:02:33.609 INFO:teuthology.orchestra.run.smithi044.stdout: , 2021-01-27T12:02:33.610 INFO:teuthology.orchestra.run.smithi044.stdout:{"epoch":102,"fsid":"3031f46c-60ee-43e8-9aa6-65c48a005092","created":"2021-01-27T12:00:07.307185+0000","modified":"2021-01-27T12:02:31.317484+0000","last_up_change":"2021-01-27T12:01:23.313632+0000","last_in_change":"2021-01-27T12:01:41.411093+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":7,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2021-01-27T12:00:11.876109+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2021-01-27T12:00:18.424507+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":18,"pg_placement_num":18,"pg_placement_num_target":18,"pg_num_target":18,"pg_num_pending":18,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"45","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"base","create_time":"2021-01-27T12:00:32.021550+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":4,"pg_placement_num":4,"pg_placement_num_target":4,"pg_num_target":4,"pg_num_pending":4,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"33","last_force_op_resend":"26","last_force_op_resend_prenautilus":"26","last_force_op_resend_preluminous":"26","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":24,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[4],"tier_of":-1,"read_tier":4,"write_tier":4,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}},{"pool":4,"pool_name":"cache","create_time":"2021-01-27T12:00:34.285167+0000","flags":9,"flags_names":"hashpspool,incomplete_clones","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":4,"pg_placement_num":4,"pg_placement_num_target":4,"pg_num_target":4,"pg_num_pending":4,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"33","last_force_op_resend":"26","last_force_op_resend_prenautilus":"26","last_force_op_resend_preluminous":"26","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":24,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":3,"read_tier":-1,"write_tier":-1,"cache_mode":"writeback","target_max_bytes":0,"target_max_objects":250,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"bloom","false_positive_probability":0.050000000000000003,"target_size":0,"seed":0},"hit_set_period":60,"hit_set_count":8,"use_gmt_hitset":true,"min_read_recency_for_promote":2,"min_write_recency_for_promote":2,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[1000000,1000000,1000000,1000000,1000000,1000000,1000000,1000000],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{}}],"osds":[{"osd":0,"uuid":"d8c64d60-e4e8-4924-8810-e49c7d811f8b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":73,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6808","nonce":35283}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6809","nonce":35283}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6811","nonce":35283}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6810","nonce":35283}]},"public_addr":"172.21.15.44:6808/35283","cluster_addr":"172.21.15.44:6809/35283","heartbeat_back_addr":"172.21.15.44:6811/35283","heartbeat_front_addr":"172.21.15.44:6810/35283","state":["exists","up"]},{"osd":1,"uuid":"2ac46988-25c0-44c7-ab6f-c8fd35b3a8cc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":101,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6804","nonce":35280}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6805","nonce":35280}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6807","nonce":35280}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6806","nonce":35280}]},"public_addr":"172.21.15.44:6804/35280","cluster_addr":"172.21.15.44:6805/35280","heartbeat_back_addr":"172.21.15.44:6807/35280","heartbeat_front_addr":"172.21.15.44:6806/35280","state":["exists","up"]},{"osd":2,"uuid":"c380bc4d-806c-44ae-97ba-7c06e3bc5533","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":67,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6812","nonce":35306}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6813","nonce":35306}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6815","nonce":35306}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6814","nonce":35306}]},"public_addr":"172.21.15.44:6812/35306","cluster_addr":"172.21.15.44:6813/35306","heartbeat_back_addr":"172.21.15.44:6815/35306","heartbeat_front_addr":"172.21.15.44:6814/35306","state":["exists","up"]},{"osd":3,"uuid":"2045e901-afee-463b-bd24-9dfdc7d28d44","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":74,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6800","nonce":35282}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6801","nonce":35282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6803","nonce":35282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6802","nonce":35282}]},"public_addr":"172.21.15.44:6800/35282","cluster_addr":"172.21.15.44:6801/35282","heartbeat_back_addr":"172.21.15.44:6803/35282","heartbeat_front_addr":"172.21.15.44:6802/35282","state":["exists","up"]},{"osd":4,"uuid":"b574f95c-e85a-4afb-9cbb-5a02f9f52253","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":73,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6808","nonce":35191}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6809","nonce":35191}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6811","nonce":35191}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6810","nonce":35191}]},"public_addr":"172.21.15.53:6808/35191","cluster_addr":"172.21.15.53:6809/35191","heartbeat_back_addr":"172.21.15.53:6811/35191","heartbeat_front_addr":"172.21.15.53:6810/35191","state":["exists","up"]},{"osd":5,"uuid":"09926f7f-9da0-4fee-9444-6f62c9353951","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":14,"last_clean_end":24,"up_from":54,"up_thru":63,"down_at":25,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6800","nonce":47032}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6801","nonce":47032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6803","nonce":47032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6802","nonce":47032}]},"public_addr":"172.21.15.53:6800/47032","cluster_addr":"172.21.15.53:6801/47032","heartbeat_back_addr":"172.21.15.53:6803/47032","heartbeat_front_addr":"172.21.15.53:6802/47032","state":["exists","up"]},{"osd":6,"uuid":"5b667ffb-d1e9-40d0-a306-d8b35218c09f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":72,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6804","nonce":35182}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6805","nonce":35182}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6807","nonce":35182}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6806","nonce":35182}]},"public_addr":"172.21.15.53:6804/35182","cluster_addr":"172.21.15.53:6805/35182","heartbeat_back_addr":"172.21.15.53:6807/35182","heartbeat_front_addr":"172.21.15.53:6806/35182","state":["exists","up"]},{"osd":7,"uuid":"15fa8e0f-4553-43a2-a3db-b6afd186a443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6812","nonce":35212}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6813","nonce":35212}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6815","nonce":35212}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6814","nonce":35212}]},"public_addr":"172.21.15.53:6812/35212","cluster_addr":"172.21.15.53:6813/35212","heartbeat_back_addr":"172.21.15.53:6815/35212","heartbeat_front_addr":"172.21.15.53:6814/35212","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:16.263237+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.506739+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.425556+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:14.953192+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.357603+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:24.772600+0000","dead_epoch":25},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:14.973919+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.916299+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.10","mappings":[{"from":2,"to":6},{"from":1,"to":0}]}],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} Ignoring malparsed backtrace: 2021-01-27T12:03:00.378 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2021-01-27T12:02:45.880587+0000 time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=45, tm_wday=2, tm_yday=27, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=27, tm_isdst=0) , 2021-01-27T12:03:00.378 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2021-01-27T12:02:36.128894+0000 time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=36, tm_wday=2, tm_yday=27, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=27, tm_isdst=0) , 2021-01-27T12:03:00.379 INFO:tasks.ceph:pgid 4.0 last_scrub_stamp 2021-01-27T12:01:52.526935+0000 time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=1, tm_sec=52, tm_wday=2, tm_yday=27, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=27, tm_isdst=0) Ignoring malparsed backtrace: 2021-01-27T12:03:42.890 DEBUG:teuthology.orchestra.run.smithi053:> sync && sudo umount -f /var/lib/ceph/osd/ceph-5 , 2021-01-27T12:03:42.978 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-6 on ubuntu@smithi053.front.sepia.ceph.com , 2021-01-27T12:03:42.978 DEBUG:teuthology.orchestra.run.smithi053:> sync && sudo umount -f /var/lib/ceph/osd/ceph-6 Ignoring malparsed backtrace: 2021-01-27T12:04:00.247 DEBUG:teuthology.orchestra.run.smithi044:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log , 2021-01-27T12:04:00.325 DEBUG:teuthology.orchestra.run.smithi053:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log , 2021-01-27T12:04:00.389 DEBUG:teuthology.orchestra.run.smithi044:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av '\(comm="dmidecode"\|chronyd.service\|name="cephtest"\|scontext=system_u:system_r:nrpe_t:s0\|scontext=system_u:system_r:pcp_pmlogger_t\|scontext=system_u:system_r:pcp_pmcd_t:s0\|comm="rhsmd"\|scontext=system_u:system_r:syslogd_t:s0\|tcontext=system_u:system_r:nrpe_t:s0\|comm="updatedb"\|comm="smartd"\|comm="rhsmcertd-worke"\|comm="setroubleshootd"\|comm="rpm"\|tcontext=system_u:object_r:container_runtime_exec_t:s0\|comm="ksmtuned"\|scontext=system_u:system_r:logrotate_t:s0\)' Ignoring malparsed backtrace: ceph version 17.0.0-184-g507028cb (507028cb839c25d6666833702df6325fd8569096) quincy (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f5cda87d15a] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x666) [0x55f42d6e2fe6] Ignoring malparsed backtrace: 2021-01-27T12:02:10.408 DEBUG:teuthology.orchestra.run.smithi044:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd reweight 0 1 , nodeep-scrub is unset , reweighted osd.0 to 1 (10000) Ignoring malparsed backtrace: 2021-01-27T12:02:33.295 DEBUG:teuthology.orchestra.run.smithi044:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2021-01-27T12:02:33.609 INFO:teuthology.orchestra.run.smithi044.stdout: , 2021-01-27T12:02:33.610 INFO:teuthology.orchestra.run.smithi044.stdout:{"epoch":102,"fsid":"3031f46c-60ee-43e8-9aa6-65c48a005092","created":"2021-01-27T12:00:07.307185+0000","modified":"2021-01-27T12:02:31.317484+0000","last_up_change":"2021-01-27T12:01:23.313632+0000","last_in_change":"2021-01-27T12:01:41.411093+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":7,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2021-01-27T12:00:11.876109+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2021-01-27T12:00:18.424507+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":18,"pg_placement_num":18,"pg_placement_num_target":18,"pg_num_target":18,"pg_num_pending":18,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"45","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"base","create_time":"2021-01-27T12:00:32.021550+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":4,"pg_placement_num":4,"pg_placement_num_target":4,"pg_num_target":4,"pg_num_pending":4,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"33","last_force_op_resend":"26","last_force_op_resend_prenautilus":"26","last_force_op_resend_preluminous":"26","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":24,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[4],"tier_of":-1,"read_tier":4,"write_tier":4,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}},{"pool":4,"pool_name":"cache","create_time":"2021-01-27T12:00:34.285167+0000","flags":9,"flags_names":"hashpspool,incomplete_clones","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":4,"pg_placement_num":4,"pg_placement_num_target":4,"pg_num_target":4,"pg_num_pending":4,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"33","last_force_op_resend":"26","last_force_op_resend_prenautilus":"26","last_force_op_resend_preluminous":"26","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":24,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":3,"read_tier":-1,"write_tier":-1,"cache_mode":"writeback","target_max_bytes":0,"target_max_objects":250,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"bloom","false_positive_probability":0.050000000000000003,"target_size":0,"seed":0},"hit_set_period":60,"hit_set_count":8,"use_gmt_hitset":true,"min_read_recency_for_promote":2,"min_write_recency_for_promote":2,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[1000000,1000000,1000000,1000000,1000000,1000000,1000000,1000000],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{}}],"osds":[{"osd":0,"uuid":"d8c64d60-e4e8-4924-8810-e49c7d811f8b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":73,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6808","nonce":35283}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6809","nonce":35283}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6811","nonce":35283}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6810","nonce":35283}]},"public_addr":"172.21.15.44:6808/35283","cluster_addr":"172.21.15.44:6809/35283","heartbeat_back_addr":"172.21.15.44:6811/35283","heartbeat_front_addr":"172.21.15.44:6810/35283","state":["exists","up"]},{"osd":1,"uuid":"2ac46988-25c0-44c7-ab6f-c8fd35b3a8cc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":101,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6804","nonce":35280}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6805","nonce":35280}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6807","nonce":35280}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6806","nonce":35280}]},"public_addr":"172.21.15.44:6804/35280","cluster_addr":"172.21.15.44:6805/35280","heartbeat_back_addr":"172.21.15.44:6807/35280","heartbeat_front_addr":"172.21.15.44:6806/35280","state":["exists","up"]},{"osd":2,"uuid":"c380bc4d-806c-44ae-97ba-7c06e3bc5533","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":67,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6812","nonce":35306}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6813","nonce":35306}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6815","nonce":35306}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6814","nonce":35306}]},"public_addr":"172.21.15.44:6812/35306","cluster_addr":"172.21.15.44:6813/35306","heartbeat_back_addr":"172.21.15.44:6815/35306","heartbeat_front_addr":"172.21.15.44:6814/35306","state":["exists","up"]},{"osd":3,"uuid":"2045e901-afee-463b-bd24-9dfdc7d28d44","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":74,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6800","nonce":35282}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6801","nonce":35282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6803","nonce":35282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6802","nonce":35282}]},"public_addr":"172.21.15.44:6800/35282","cluster_addr":"172.21.15.44:6801/35282","heartbeat_back_addr":"172.21.15.44:6803/35282","heartbeat_front_addr":"172.21.15.44:6802/35282","state":["exists","up"]},{"osd":4,"uuid":"b574f95c-e85a-4afb-9cbb-5a02f9f52253","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":73,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6808","nonce":35191}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6809","nonce":35191}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6811","nonce":35191}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6810","nonce":35191}]},"public_addr":"172.21.15.53:6808/35191","cluster_addr":"172.21.15.53:6809/35191","heartbeat_back_addr":"172.21.15.53:6811/35191","heartbeat_front_addr":"172.21.15.53:6810/35191","state":["exists","up"]},{"osd":5,"uuid":"09926f7f-9da0-4fee-9444-6f62c9353951","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":14,"last_clean_end":24,"up_from":54,"up_thru":63,"down_at":25,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6800","nonce":47032}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6801","nonce":47032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6803","nonce":47032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6802","nonce":47032}]},"public_addr":"172.21.15.53:6800/47032","cluster_addr":"172.21.15.53:6801/47032","heartbeat_back_addr":"172.21.15.53:6803/47032","heartbeat_front_addr":"172.21.15.53:6802/47032","state":["exists","up"]},{"osd":6,"uuid":"5b667ffb-d1e9-40d0-a306-d8b35218c09f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":72,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6804","nonce":35182}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6805","nonce":35182}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6807","nonce":35182}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6806","nonce":35182}]},"public_addr":"172.21.15.53:6804/35182","cluster_addr":"172.21.15.53:6805/35182","heartbeat_back_addr":"172.21.15.53:6807/35182","heartbeat_front_addr":"172.21.15.53:6806/35182","state":["exists","up"]},{"osd":7,"uuid":"15fa8e0f-4553-43a2-a3db-b6afd186a443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6812","nonce":35212}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6813","nonce":35212}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6815","nonce":35212}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6814","nonce":35212}]},"public_addr":"172.21.15.53:6812/35212","cluster_addr":"172.21.15.53:6813/35212","heartbeat_back_addr":"172.21.15.53:6815/35212","heartbeat_front_addr":"172.21.15.53:6814/35212","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:16.263237+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.506739+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.425556+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:14.953192+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.357603+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:24.772600+0000","dead_epoch":25},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:14.973919+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.916299+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.10","mappings":[{"from":2,"to":6},{"from":1,"to":0}]}],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} Ignoring malparsed backtrace: 2021-01-27T12:03:00.378 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2021-01-27T12:02:45.880587+0000 time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=45, tm_wday=2, tm_yday=27, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=27, tm_isdst=0) , 2021-01-27T12:03:00.378 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2021-01-27T12:02:36.128894+0000 time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=36, tm_wday=2, tm_yday=27, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=27, tm_isdst=0) , 2021-01-27T12:03:00.379 INFO:tasks.ceph:pgid 4.0 last_scrub_stamp 2021-01-27T12:01:52.526935+0000 time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=1, tm_sec=52, tm_wday=2, tm_yday=27, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=27, tm_isdst=0) Ignoring malparsed backtrace: 2021-01-27T12:03:42.890 DEBUG:teuthology.orchestra.run.smithi053:> sync && sudo umount -f /var/lib/ceph/osd/ceph-5 , 2021-01-27T12:03:42.978 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-6 on ubuntu@smithi053.front.sepia.ceph.com , 2021-01-27T12:03:42.978 DEBUG:teuthology.orchestra.run.smithi053:> sync && sudo umount -f /var/lib/ceph/osd/ceph-6 Ignoring malparsed backtrace: 2021-01-27T12:04:00.247 DEBUG:teuthology.orchestra.run.smithi044:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log , 2021-01-27T12:04:00.325 DEBUG:teuthology.orchestra.run.smithi053:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log , 2021-01-27T12:04:00.389 DEBUG:teuthology.orchestra.run.smithi044:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av '\(comm="dmidecode"\|chronyd.service\|name="cephtest"\|scontext=system_u:system_r:nrpe_t:s0\|scontext=system_u:system_r:pcp_pmlogger_t\|scontext=system_u:system_r:pcp_pmcd_t:s0\|comm="rhsmd"\|scontext=system_u:system_r:syslogd_t:s0\|tcontext=system_u:system_r:nrpe_t:s0\|comm="updatedb"\|comm="smartd"\|comm="rhsmcertd-worke"\|comm="setroubleshootd"\|comm="rpm"\|tcontext=system_u:object_r:container_runtime_exec_t:s0\|comm="ksmtuned"\|scontext=system_u:system_r:logrotate_t:s0\)' Ignoring malparsed backtrace: ceph version 17.0.0-184-g507028cb (507028cb839c25d6666833702df6325fd8569096) quincy (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f5cda87d15a] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x666) [0x55f42d6e2fe6] Ignoring malparsed backtrace: 2021-01-27T12:02:10.408 DEBUG:teuthology.orchestra.run.smithi044:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd reweight 0 1 , nodeep-scrub is unset , reweighted osd.0 to 1 (10000) Ignoring malparsed backtrace: 2021-01-27T12:02:33.295 DEBUG:teuthology.orchestra.run.smithi044:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2021-01-27T12:02:33.609 INFO:teuthology.orchestra.run.smithi044.stdout: , 2021-01-27T12:02:33.610 INFO:teuthology.orchestra.run.smithi044.stdout:{"epoch":102,"fsid":"3031f46c-60ee-43e8-9aa6-65c48a005092","created":"2021-01-27T12:00:07.307185+0000","modified":"2021-01-27T12:02:31.317484+0000","last_up_change":"2021-01-27T12:01:23.313632+0000","last_in_change":"2021-01-27T12:01:41.411093+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":7,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2021-01-27T12:00:11.876109+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2021-01-27T12:00:18.424507+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":18,"pg_placement_num":18,"pg_placement_num_target":18,"pg_num_target":18,"pg_num_pending":18,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"45","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"base","create_time":"2021-01-27T12:00:32.021550+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":4,"pg_placement_num":4,"pg_placement_num_target":4,"pg_num_target":4,"pg_num_pending":4,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"33","last_force_op_resend":"26","last_force_op_resend_prenautilus":"26","last_force_op_resend_preluminous":"26","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":24,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[4],"tier_of":-1,"read_tier":4,"write_tier":4,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}},{"pool":4,"pool_name":"cache","create_time":"2021-01-27T12:00:34.285167+0000","flags":9,"flags_names":"hashpspool,incomplete_clones","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":4,"pg_placement_num":4,"pg_placement_num_target":4,"pg_num_target":4,"pg_num_pending":4,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"33","last_force_op_resend":"26","last_force_op_resend_prenautilus":"26","last_force_op_resend_preluminous":"26","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":24,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":3,"read_tier":-1,"write_tier":-1,"cache_mode":"writeback","target_max_bytes":0,"target_max_objects":250,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"bloom","false_positive_probability":0.050000000000000003,"target_size":0,"seed":0},"hit_set_period":60,"hit_set_count":8,"use_gmt_hitset":true,"min_read_recency_for_promote":2,"min_write_recency_for_promote":2,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[1000000,1000000,1000000,1000000,1000000,1000000,1000000,1000000],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{}}],"osds":[{"osd":0,"uuid":"d8c64d60-e4e8-4924-8810-e49c7d811f8b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":73,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6808","nonce":35283}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6809","nonce":35283}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6811","nonce":35283}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6810","nonce":35283}]},"public_addr":"172.21.15.44:6808/35283","cluster_addr":"172.21.15.44:6809/35283","heartbeat_back_addr":"172.21.15.44:6811/35283","heartbeat_front_addr":"172.21.15.44:6810/35283","state":["exists","up"]},{"osd":1,"uuid":"2ac46988-25c0-44c7-ab6f-c8fd35b3a8cc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":101,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6804","nonce":35280}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6805","nonce":35280}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6807","nonce":35280}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6806","nonce":35280}]},"public_addr":"172.21.15.44:6804/35280","cluster_addr":"172.21.15.44:6805/35280","heartbeat_back_addr":"172.21.15.44:6807/35280","heartbeat_front_addr":"172.21.15.44:6806/35280","state":["exists","up"]},{"osd":2,"uuid":"c380bc4d-806c-44ae-97ba-7c06e3bc5533","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":67,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6812","nonce":35306}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6813","nonce":35306}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6815","nonce":35306}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6814","nonce":35306}]},"public_addr":"172.21.15.44:6812/35306","cluster_addr":"172.21.15.44:6813/35306","heartbeat_back_addr":"172.21.15.44:6815/35306","heartbeat_front_addr":"172.21.15.44:6814/35306","state":["exists","up"]},{"osd":3,"uuid":"2045e901-afee-463b-bd24-9dfdc7d28d44","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":74,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6800","nonce":35282}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6801","nonce":35282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6803","nonce":35282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6802","nonce":35282}]},"public_addr":"172.21.15.44:6800/35282","cluster_addr":"172.21.15.44:6801/35282","heartbeat_back_addr":"172.21.15.44:6803/35282","heartbeat_front_addr":"172.21.15.44:6802/35282","state":["exists","up"]},{"osd":4,"uuid":"b574f95c-e85a-4afb-9cbb-5a02f9f52253","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":73,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6808","nonce":35191}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6809","nonce":35191}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6811","nonce":35191}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6810","nonce":35191}]},"public_addr":"172.21.15.53:6808/35191","cluster_addr":"172.21.15.53:6809/35191","heartbeat_back_addr":"172.21.15.53:6811/35191","heartbeat_front_addr":"172.21.15.53:6810/35191","state":["exists","up"]},{"osd":5,"uuid":"09926f7f-9da0-4fee-9444-6f62c9353951","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":14,"last_clean_end":24,"up_from":54,"up_thru":63,"down_at":25,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6800","nonce":47032}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6801","nonce":47032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6803","nonce":47032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6802","nonce":47032}]},"public_addr":"172.21.15.53:6800/47032","cluster_addr":"172.21.15.53:6801/47032","heartbeat_back_addr":"172.21.15.53:6803/47032","heartbeat_front_addr":"172.21.15.53:6802/47032","state":["exists","up"]},{"osd":6,"uuid":"5b667ffb-d1e9-40d0-a306-d8b35218c09f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":72,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6804","nonce":35182}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6805","nonce":35182}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6807","nonce":35182}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6806","nonce":35182}]},"public_addr":"172.21.15.53:6804/35182","cluster_addr":"172.21.15.53:6805/35182","heartbeat_back_addr":"172.21.15.53:6807/35182","heartbeat_front_addr":"172.21.15.53:6806/35182","state":["exists","up"]},{"osd":7,"uuid":"15fa8e0f-4553-43a2-a3db-b6afd186a443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6812","nonce":35212}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6813","nonce":35212}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6815","nonce":35212}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6814","nonce":35212}]},"public_addr":"172.21.15.53:6812/35212","cluster_addr":"172.21.15.53:6813/35212","heartbeat_back_addr":"172.21.15.53:6815/35212","heartbeat_front_addr":"172.21.15.53:6814/35212","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:16.263237+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.506739+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.425556+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:14.953192+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.357603+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:24.772600+0000","dead_epoch":25},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:14.973919+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.916299+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.10","mappings":[{"from":2,"to":6},{"from":1,"to":0}]}],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} Ignoring malparsed backtrace: 2021-01-27T12:03:00.378 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2021-01-27T12:02:45.880587+0000 time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=45, tm_wday=2, tm_yday=27, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=27, tm_isdst=0) , 2021-01-27T12:03:00.378 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2021-01-27T12:02:36.128894+0000 time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=36, tm_wday=2, tm_yday=27, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=27, tm_isdst=0) , 2021-01-27T12:03:00.379 INFO:tasks.ceph:pgid 4.0 last_scrub_stamp 2021-01-27T12:01:52.526935+0000 time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=1, tm_sec=52, tm_wday=2, tm_yday=27, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=27, tm_isdst=0) Ignoring malparsed backtrace: 2021-01-27T12:03:42.890 DEBUG:teuthology.orchestra.run.smithi053:> sync && sudo umount -f /var/lib/ceph/osd/ceph-5 , 2021-01-27T12:03:42.978 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-6 on ubuntu@smithi053.front.sepia.ceph.com , 2021-01-27T12:03:42.978 DEBUG:teuthology.orchestra.run.smithi053:> sync && sudo umount -f /var/lib/ceph/osd/ceph-6 Ignoring malparsed backtrace: 2021-01-27T12:04:00.247 DEBUG:teuthology.orchestra.run.smithi044:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log , 2021-01-27T12:04:00.325 DEBUG:teuthology.orchestra.run.smithi053:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log , 2021-01-27T12:04:00.389 DEBUG:teuthology.orchestra.run.smithi044:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av '\(comm="dmidecode"\|chronyd.service\|name="cephtest"\|scontext=system_u:system_r:nrpe_t:s0\|scontext=system_u:system_r:pcp_pmlogger_t\|scontext=system_u:system_r:pcp_pmcd_t:s0\|comm="rhsmd"\|scontext=system_u:system_r:syslogd_t:s0\|tcontext=system_u:system_r:nrpe_t:s0\|comm="updatedb"\|comm="smartd"\|comm="rhsmcertd-worke"\|comm="setroubleshootd"\|comm="rpm"\|tcontext=system_u:object_r:container_runtime_exec_t:s0\|comm="ksmtuned"\|scontext=system_u:system_r:logrotate_t:s0\)' Ignoring malparsed backtrace: ceph version 17.0.0-184-g507028cb (507028cb839c25d6666833702df6325fd8569096) quincy (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f5cda87d15a] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x666) [0x55f42d6e2fe6] Ignoring malparsed backtrace: 2021-01-27T12:02:10.408 DEBUG:teuthology.orchestra.run.smithi044:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd reweight 0 1 , nodeep-scrub is unset , reweighted osd.0 to 1 (10000) Ignoring malparsed backtrace: 2021-01-27T12:02:33.295 DEBUG:teuthology.orchestra.run.smithi044:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2021-01-27T12:02:33.609 INFO:teuthology.orchestra.run.smithi044.stdout: , 2021-01-27T12:02:33.610 INFO:teuthology.orchestra.run.smithi044.stdout:{"epoch":102,"fsid":"3031f46c-60ee-43e8-9aa6-65c48a005092","created":"2021-01-27T12:00:07.307185+0000","modified":"2021-01-27T12:02:31.317484+0000","last_up_change":"2021-01-27T12:01:23.313632+0000","last_in_change":"2021-01-27T12:01:41.411093+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":7,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2021-01-27T12:00:11.876109+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2021-01-27T12:00:18.424507+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":18,"pg_placement_num":18,"pg_placement_num_target":18,"pg_num_target":18,"pg_num_pending":18,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"45","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"base","create_time":"2021-01-27T12:00:32.021550+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":4,"pg_placement_num":4,"pg_placement_num_target":4,"pg_num_target":4,"pg_num_pending":4,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"33","last_force_op_resend":"26","last_force_op_resend_prenautilus":"26","last_force_op_resend_preluminous":"26","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":24,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[4],"tier_of":-1,"read_tier":4,"write_tier":4,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}},{"pool":4,"pool_name":"cache","create_time":"2021-01-27T12:00:34.285167+0000","flags":9,"flags_names":"hashpspool,incomplete_clones","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":4,"pg_placement_num":4,"pg_placement_num_target":4,"pg_num_target":4,"pg_num_pending":4,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"33","last_force_op_resend":"26","last_force_op_resend_prenautilus":"26","last_force_op_resend_preluminous":"26","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":24,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":3,"read_tier":-1,"write_tier":-1,"cache_mode":"writeback","target_max_bytes":0,"target_max_objects":250,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"bloom","false_positive_probability":0.050000000000000003,"target_size":0,"seed":0},"hit_set_period":60,"hit_set_count":8,"use_gmt_hitset":true,"min_read_recency_for_promote":2,"min_write_recency_for_promote":2,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[1000000,1000000,1000000,1000000,1000000,1000000,1000000,1000000],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{}}],"osds":[{"osd":0,"uuid":"d8c64d60-e4e8-4924-8810-e49c7d811f8b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":73,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6808","nonce":35283}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6809","nonce":35283}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6811","nonce":35283}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6810","nonce":35283}]},"public_addr":"172.21.15.44:6808/35283","cluster_addr":"172.21.15.44:6809/35283","heartbeat_back_addr":"172.21.15.44:6811/35283","heartbeat_front_addr":"172.21.15.44:6810/35283","state":["exists","up"]},{"osd":1,"uuid":"2ac46988-25c0-44c7-ab6f-c8fd35b3a8cc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":101,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6804","nonce":35280}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6805","nonce":35280}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6807","nonce":35280}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6806","nonce":35280}]},"public_addr":"172.21.15.44:6804/35280","cluster_addr":"172.21.15.44:6805/35280","heartbeat_back_addr":"172.21.15.44:6807/35280","heartbeat_front_addr":"172.21.15.44:6806/35280","state":["exists","up"]},{"osd":2,"uuid":"c380bc4d-806c-44ae-97ba-7c06e3bc5533","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":67,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6812","nonce":35306}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6813","nonce":35306}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6815","nonce":35306}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6814","nonce":35306}]},"public_addr":"172.21.15.44:6812/35306","cluster_addr":"172.21.15.44:6813/35306","heartbeat_back_addr":"172.21.15.44:6815/35306","heartbeat_front_addr":"172.21.15.44:6814/35306","state":["exists","up"]},{"osd":3,"uuid":"2045e901-afee-463b-bd24-9dfdc7d28d44","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":74,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6800","nonce":35282}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6801","nonce":35282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6803","nonce":35282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6802","nonce":35282}]},"public_addr":"172.21.15.44:6800/35282","cluster_addr":"172.21.15.44:6801/35282","heartbeat_back_addr":"172.21.15.44:6803/35282","heartbeat_front_addr":"172.21.15.44:6802/35282","state":["exists","up"]},{"osd":4,"uuid":"b574f95c-e85a-4afb-9cbb-5a02f9f52253","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":73,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6808","nonce":35191}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6809","nonce":35191}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6811","nonce":35191}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6810","nonce":35191}]},"public_addr":"172.21.15.53:6808/35191","cluster_addr":"172.21.15.53:6809/35191","heartbeat_back_addr":"172.21.15.53:6811/35191","heartbeat_front_addr":"172.21.15.53:6810/35191","state":["exists","up"]},{"osd":5,"uuid":"09926f7f-9da0-4fee-9444-6f62c9353951","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":14,"last_clean_end":24,"up_from":54,"up_thru":63,"down_at":25,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6800","nonce":47032}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6801","nonce":47032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6803","nonce":47032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6802","nonce":47032}]},"public_addr":"172.21.15.53:6800/47032","cluster_addr":"172.21.15.53:6801/47032","heartbeat_back_addr":"172.21.15.53:6803/47032","heartbeat_front_addr":"172.21.15.53:6802/47032","state":["exists","up"]},{"osd":6,"uuid":"5b667ffb-d1e9-40d0-a306-d8b35218c09f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":72,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6804","nonce":35182}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6805","nonce":35182}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6807","nonce":35182}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6806","nonce":35182}]},"public_addr":"172.21.15.53:6804/35182","cluster_addr":"172.21.15.53:6805/35182","heartbeat_back_addr":"172.21.15.53:6807/35182","heartbeat_front_addr":"172.21.15.53:6806/35182","state":["exists","up"]},{"osd":7,"uuid":"15fa8e0f-4553-43a2-a3db-b6afd186a443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6812","nonce":35212}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6813","nonce":35212}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6815","nonce":35212}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6814","nonce":35212}]},"public_addr":"172.21.15.53:6812/35212","cluster_addr":"172.21.15.53:6813/35212","heartbeat_back_addr":"172.21.15.53:6815/35212","heartbeat_front_addr":"172.21.15.53:6814/35212","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:16.263237+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.506739+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.425556+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:14.953192+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.357603+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:24.772600+0000","dead_epoch":25},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:14.973919+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.916299+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.10","mappings":[{"from":2,"to":6},{"from":1,"to":0}]}],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} Ignoring malparsed backtrace: 2021-01-27T12:03:00.378 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2021-01-27T12:02:45.880587+0000 time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=45, tm_wday=2, tm_yday=27, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=27, tm_isdst=0) , 2021-01-27T12:03:00.378 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2021-01-27T12:02:36.128894+0000 time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=36, tm_wday=2, tm_yday=27, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=27, tm_isdst=0) , 2021-01-27T12:03:00.379 INFO:tasks.ceph:pgid 4.0 last_scrub_stamp 2021-01-27T12:01:52.526935+0000 time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=1, tm_sec=52, tm_wday=2, tm_yday=27, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=27, tm_isdst=0) Ignoring malparsed backtrace: 2021-01-27T12:03:42.890 DEBUG:teuthology.orchestra.run.smithi053:> sync && sudo umount -f /var/lib/ceph/osd/ceph-5 , 2021-01-27T12:03:42.978 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-6 on ubuntu@smithi053.front.sepia.ceph.com , 2021-01-27T12:03:42.978 DEBUG:teuthology.orchestra.run.smithi053:> sync && sudo umount -f /var/lib/ceph/osd/ceph-6 Ignoring malparsed backtrace: 2021-01-27T12:04:00.247 DEBUG:teuthology.orchestra.run.smithi044:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log , 2021-01-27T12:04:00.325 DEBUG:teuthology.orchestra.run.smithi053:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log , 2021-01-27T12:04:00.389 DEBUG:teuthology.orchestra.run.smithi044:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av '\(comm="dmidecode"\|chronyd.service\|name="cephtest"\|scontext=system_u:system_r:nrpe_t:s0\|scontext=system_u:system_r:pcp_pmlogger_t\|scontext=system_u:system_r:pcp_pmcd_t:s0\|comm="rhsmd"\|scontext=system_u:system_r:syslogd_t:s0\|tcontext=system_u:system_r:nrpe_t:s0\|comm="updatedb"\|comm="smartd"\|comm="rhsmcertd-worke"\|comm="setroubleshootd"\|comm="rpm"\|tcontext=system_u:object_r:container_runtime_exec_t:s0\|comm="ksmtuned"\|scontext=system_u:system_r:logrotate_t:s0\)' Ignoring malparsed backtrace: ceph version 17.0.0-184-g507028cb (507028cb839c25d6666833702df6325fd8569096) quincy (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f5cda87d15a] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x666) [0x55f42d6e2fe6] Ignoring malparsed backtrace: 2021-01-27T12:02:10.408 DEBUG:teuthology.orchestra.run.smithi044:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd reweight 0 1 , nodeep-scrub is unset , reweighted osd.0 to 1 (10000) Ignoring malparsed backtrace: 2021-01-27T12:02:33.295 DEBUG:teuthology.orchestra.run.smithi044:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2021-01-27T12:02:33.609 INFO:teuthology.orchestra.run.smithi044.stdout: , 2021-01-27T12:02:33.610 INFO:teuthology.orchestra.run.smithi044.stdout:{"epoch":102,"fsid":"3031f46c-60ee-43e8-9aa6-65c48a005092","created":"2021-01-27T12:00:07.307185+0000","modified":"2021-01-27T12:02:31.317484+0000","last_up_change":"2021-01-27T12:01:23.313632+0000","last_in_change":"2021-01-27T12:01:41.411093+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":7,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2021-01-27T12:00:11.876109+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2021-01-27T12:00:18.424507+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":18,"pg_placement_num":18,"pg_placement_num_target":18,"pg_num_target":18,"pg_num_pending":18,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"45","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"base","create_time":"2021-01-27T12:00:32.021550+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":4,"pg_placement_num":4,"pg_placement_num_target":4,"pg_num_target":4,"pg_num_pending":4,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"33","last_force_op_resend":"26","last_force_op_resend_prenautilus":"26","last_force_op_resend_preluminous":"26","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":24,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[4],"tier_of":-1,"read_tier":4,"write_tier":4,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}},{"pool":4,"pool_name":"cache","create_time":"2021-01-27T12:00:34.285167+0000","flags":9,"flags_names":"hashpspool,incomplete_clones","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":4,"pg_placement_num":4,"pg_placement_num_target":4,"pg_num_target":4,"pg_num_pending":4,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"33","last_force_op_resend":"26","last_force_op_resend_prenautilus":"26","last_force_op_resend_preluminous":"26","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":24,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":3,"read_tier":-1,"write_tier":-1,"cache_mode":"writeback","target_max_bytes":0,"target_max_objects":250,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"bloom","false_positive_probability":0.050000000000000003,"target_size":0,"seed":0},"hit_set_period":60,"hit_set_count":8,"use_gmt_hitset":true,"min_read_recency_for_promote":2,"min_write_recency_for_promote":2,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[1000000,1000000,1000000,1000000,1000000,1000000,1000000,1000000],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{}}],"osds":[{"osd":0,"uuid":"d8c64d60-e4e8-4924-8810-e49c7d811f8b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":73,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6808","nonce":35283}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6809","nonce":35283}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6811","nonce":35283}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6810","nonce":35283}]},"public_addr":"172.21.15.44:6808/35283","cluster_addr":"172.21.15.44:6809/35283","heartbeat_back_addr":"172.21.15.44:6811/35283","heartbeat_front_addr":"172.21.15.44:6810/35283","state":["exists","up"]},{"osd":1,"uuid":"2ac46988-25c0-44c7-ab6f-c8fd35b3a8cc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":101,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6804","nonce":35280}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6805","nonce":35280}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6807","nonce":35280}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6806","nonce":35280}]},"public_addr":"172.21.15.44:6804/35280","cluster_addr":"172.21.15.44:6805/35280","heartbeat_back_addr":"172.21.15.44:6807/35280","heartbeat_front_addr":"172.21.15.44:6806/35280","state":["exists","up"]},{"osd":2,"uuid":"c380bc4d-806c-44ae-97ba-7c06e3bc5533","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":67,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6812","nonce":35306}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6813","nonce":35306}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6815","nonce":35306}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6814","nonce":35306}]},"public_addr":"172.21.15.44:6812/35306","cluster_addr":"172.21.15.44:6813/35306","heartbeat_back_addr":"172.21.15.44:6815/35306","heartbeat_front_addr":"172.21.15.44:6814/35306","state":["exists","up"]},{"osd":3,"uuid":"2045e901-afee-463b-bd24-9dfdc7d28d44","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":74,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6800","nonce":35282}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6801","nonce":35282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6803","nonce":35282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6802","nonce":35282}]},"public_addr":"172.21.15.44:6800/35282","cluster_addr":"172.21.15.44:6801/35282","heartbeat_back_addr":"172.21.15.44:6803/35282","heartbeat_front_addr":"172.21.15.44:6802/35282","state":["exists","up"]},{"osd":4,"uuid":"b574f95c-e85a-4afb-9cbb-5a02f9f52253","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":73,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6808","nonce":35191}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6809","nonce":35191}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6811","nonce":35191}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6810","nonce":35191}]},"public_addr":"172.21.15.53:6808/35191","cluster_addr":"172.21.15.53:6809/35191","heartbeat_back_addr":"172.21.15.53:6811/35191","heartbeat_front_addr":"172.21.15.53:6810/35191","state":["exists","up"]},{"osd":5,"uuid":"09926f7f-9da0-4fee-9444-6f62c9353951","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":14,"last_clean_end":24,"up_from":54,"up_thru":63,"down_at":25,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6800","nonce":47032}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6801","nonce":47032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6803","nonce":47032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6802","nonce":47032}]},"public_addr":"172.21.15.53:6800/47032","cluster_addr":"172.21.15.53:6801/47032","heartbeat_back_addr":"172.21.15.53:6803/47032","heartbeat_front_addr":"172.21.15.53:6802/47032","state":["exists","up"]},{"osd":6,"uuid":"5b667ffb-d1e9-40d0-a306-d8b35218c09f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":72,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6804","nonce":35182}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6805","nonce":35182}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6807","nonce":35182}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6806","nonce":35182}]},"public_addr":"172.21.15.53:6804/35182","cluster_addr":"172.21.15.53:6805/35182","heartbeat_back_addr":"172.21.15.53:6807/35182","heartbeat_front_addr":"172.21.15.53:6806/35182","state":["exists","up"]},{"osd":7,"uuid":"15fa8e0f-4553-43a2-a3db-b6afd186a443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6812","nonce":35212}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6813","nonce":35212}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6815","nonce":35212}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6814","nonce":35212}]},"public_addr":"172.21.15.53:6812/35212","cluster_addr":"172.21.15.53:6813/35212","heartbeat_back_addr":"172.21.15.53:6815/35212","heartbeat_front_addr":"172.21.15.53:6814/35212","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:16.263237+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.506739+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.425556+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:14.953192+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.357603+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:24.772600+0000","dead_epoch":25},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:14.973919+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.916299+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.10","mappings":[{"from":2,"to":6},{"from":1,"to":0}]}],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} Ignoring malparsed backtrace: 2021-01-27T12:03:00.378 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2021-01-27T12:02:45.880587+0000 time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=45, tm_wday=2, tm_yday=27, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=27, tm_isdst=0) , 2021-01-27T12:03:00.378 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2021-01-27T12:02:36.128894+0000 time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=36, tm_wday=2, tm_yday=27, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=27, tm_isdst=0) , 2021-01-27T12:03:00.379 INFO:tasks.ceph:pgid 4.0 last_scrub_stamp 2021-01-27T12:01:52.526935+0000 time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=1, tm_sec=52, tm_wday=2, tm_yday=27, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=27, tm_isdst=0) Ignoring malparsed backtrace: 2021-01-27T12:03:42.890 DEBUG:teuthology.orchestra.run.smithi053:> sync && sudo umount -f /var/lib/ceph/osd/ceph-5 , 2021-01-27T12:03:42.978 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-6 on ubuntu@smithi053.front.sepia.ceph.com , 2021-01-27T12:03:42.978 DEBUG:teuthology.orchestra.run.smithi053:> sync && sudo umount -f /var/lib/ceph/osd/ceph-6 Ignoring malparsed backtrace: 2021-01-27T12:04:00.247 DEBUG:teuthology.orchestra.run.smithi044:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log , 2021-01-27T12:04:00.325 DEBUG:teuthology.orchestra.run.smithi053:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log , 2021-01-27T12:04:00.389 DEBUG:teuthology.orchestra.run.smithi044:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av '\(comm="dmidecode"\|chronyd.service\|name="cephtest"\|scontext=system_u:system_r:nrpe_t:s0\|scontext=system_u:system_r:pcp_pmlogger_t\|scontext=system_u:system_r:pcp_pmcd_t:s0\|comm="rhsmd"\|scontext=system_u:system_r:syslogd_t:s0\|tcontext=system_u:system_r:nrpe_t:s0\|comm="updatedb"\|comm="smartd"\|comm="rhsmcertd-worke"\|comm="setroubleshootd"\|comm="rpm"\|tcontext=system_u:object_r:container_runtime_exec_t:s0\|comm="ksmtuned"\|scontext=system_u:system_r:logrotate_t:s0\)' Ignoring malparsed backtrace: ceph version 17.0.0-184-g507028cb (507028cb839c25d6666833702df6325fd8569096) quincy (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f5cda87d15a] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x666) [0x55f42d6e2fe6] Ignoring malparsed backtrace: 2021-01-27T12:02:10.408 DEBUG:teuthology.orchestra.run.smithi044:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd reweight 0 1 , nodeep-scrub is unset , reweighted osd.0 to 1 (10000) Ignoring malparsed backtrace: 2021-01-27T12:02:33.295 DEBUG:teuthology.orchestra.run.smithi044:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json , 2021-01-27T12:02:33.609 INFO:teuthology.orchestra.run.smithi044.stdout: , 2021-01-27T12:02:33.610 INFO:teuthology.orchestra.run.smithi044.stdout:{"epoch":102,"fsid":"3031f46c-60ee-43e8-9aa6-65c48a005092","created":"2021-01-27T12:00:07.307185+0000","modified":"2021-01-27T12:02:31.317484+0000","last_up_change":"2021-01-27T12:01:23.313632+0000","last_in_change":"2021-01-27T12:01:41.411093+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":7,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2021-01-27T12:00:11.876109+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2021-01-27T12:00:18.424507+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":18,"pg_placement_num":18,"pg_placement_num_target":18,"pg_num_target":18,"pg_num_pending":18,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"45","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"base","create_time":"2021-01-27T12:00:32.021550+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":4,"pg_placement_num":4,"pg_placement_num_target":4,"pg_num_target":4,"pg_num_pending":4,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"33","last_force_op_resend":"26","last_force_op_resend_prenautilus":"26","last_force_op_resend_preluminous":"26","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":24,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[4],"tier_of":-1,"read_tier":4,"write_tier":4,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}},{"pool":4,"pool_name":"cache","create_time":"2021-01-27T12:00:34.285167+0000","flags":9,"flags_names":"hashpspool,incomplete_clones","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":4,"pg_placement_num":4,"pg_placement_num_target":4,"pg_num_target":4,"pg_num_pending":4,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"33","last_force_op_resend":"26","last_force_op_resend_prenautilus":"26","last_force_op_resend_preluminous":"26","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":24,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":3,"read_tier":-1,"write_tier":-1,"cache_mode":"writeback","target_max_bytes":0,"target_max_objects":250,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"bloom","false_positive_probability":0.050000000000000003,"target_size":0,"seed":0},"hit_set_period":60,"hit_set_count":8,"use_gmt_hitset":true,"min_read_recency_for_promote":2,"min_write_recency_for_promote":2,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[1000000,1000000,1000000,1000000,1000000,1000000,1000000,1000000],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{}}],"osds":[{"osd":0,"uuid":"d8c64d60-e4e8-4924-8810-e49c7d811f8b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":73,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6808","nonce":35283}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6809","nonce":35283}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6811","nonce":35283}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6810","nonce":35283}]},"public_addr":"172.21.15.44:6808/35283","cluster_addr":"172.21.15.44:6809/35283","heartbeat_back_addr":"172.21.15.44:6811/35283","heartbeat_front_addr":"172.21.15.44:6810/35283","state":["exists","up"]},{"osd":1,"uuid":"2ac46988-25c0-44c7-ab6f-c8fd35b3a8cc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":101,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6804","nonce":35280}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6805","nonce":35280}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6807","nonce":35280}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6806","nonce":35280}]},"public_addr":"172.21.15.44:6804/35280","cluster_addr":"172.21.15.44:6805/35280","heartbeat_back_addr":"172.21.15.44:6807/35280","heartbeat_front_addr":"172.21.15.44:6806/35280","state":["exists","up"]},{"osd":2,"uuid":"c380bc4d-806c-44ae-97ba-7c06e3bc5533","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":67,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6812","nonce":35306}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6813","nonce":35306}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6815","nonce":35306}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6814","nonce":35306}]},"public_addr":"172.21.15.44:6812/35306","cluster_addr":"172.21.15.44:6813/35306","heartbeat_back_addr":"172.21.15.44:6815/35306","heartbeat_front_addr":"172.21.15.44:6814/35306","state":["exists","up"]},{"osd":3,"uuid":"2045e901-afee-463b-bd24-9dfdc7d28d44","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":74,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6800","nonce":35282}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6801","nonce":35282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6803","nonce":35282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.44:6802","nonce":35282}]},"public_addr":"172.21.15.44:6800/35282","cluster_addr":"172.21.15.44:6801/35282","heartbeat_back_addr":"172.21.15.44:6803/35282","heartbeat_front_addr":"172.21.15.44:6802/35282","state":["exists","up"]},{"osd":4,"uuid":"b574f95c-e85a-4afb-9cbb-5a02f9f52253","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":73,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6808","nonce":35191}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6809","nonce":35191}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6811","nonce":35191}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6810","nonce":35191}]},"public_addr":"172.21.15.53:6808/35191","cluster_addr":"172.21.15.53:6809/35191","heartbeat_back_addr":"172.21.15.53:6811/35191","heartbeat_front_addr":"172.21.15.53:6810/35191","state":["exists","up"]},{"osd":5,"uuid":"09926f7f-9da0-4fee-9444-6f62c9353951","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":14,"last_clean_end":24,"up_from":54,"up_thru":63,"down_at":25,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6800","nonce":47032}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6801","nonce":47032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6803","nonce":47032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6802","nonce":47032}]},"public_addr":"172.21.15.53:6800/47032","cluster_addr":"172.21.15.53:6801/47032","heartbeat_back_addr":"172.21.15.53:6803/47032","heartbeat_front_addr":"172.21.15.53:6802/47032","state":["exists","up"]},{"osd":6,"uuid":"5b667ffb-d1e9-40d0-a306-d8b35218c09f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":72,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6804","nonce":35182}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6805","nonce":35182}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6807","nonce":35182}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6806","nonce":35182}]},"public_addr":"172.21.15.53:6804/35182","cluster_addr":"172.21.15.53:6805/35182","heartbeat_back_addr":"172.21.15.53:6807/35182","heartbeat_front_addr":"172.21.15.53:6806/35182","state":["exists","up"]},{"osd":7,"uuid":"15fa8e0f-4553-43a2-a3db-b6afd186a443","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":15,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6812","nonce":35212}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6813","nonce":35212}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6815","nonce":35212}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.53:6814","nonce":35212}]},"public_addr":"172.21.15.53:6812/35212","cluster_addr":"172.21.15.53:6813/35212","heartbeat_back_addr":"172.21.15.53:6815/35212","heartbeat_front_addr":"172.21.15.53:6814/35212","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:16.263237+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.506739+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.425556+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:14.953192+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.357603+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:24.772600+0000","dead_epoch":25},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:14.973919+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2021-01-27T12:02:15.916299+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.10","mappings":[{"from":2,"to":6},{"from":1,"to":0}]}],"pg_temp":[],"primary_temp":[],"blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} Ignoring malparsed backtrace: 2021-01-27T12:03:00.378 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2021-01-27T12:02:45.880587+0000 time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=45, tm_wday=2, tm_yday=27, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=27, tm_isdst=0) , 2021-01-27T12:03:00.378 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2021-01-27T12:02:36.128894+0000 time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=36, tm_wday=2, tm_yday=27, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=27, tm_isdst=0) , 2021-01-27T12:03:00.379 INFO:tasks.ceph:pgid 4.0 last_scrub_stamp 2021-01-27T12:01:52.526935+0000 time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=1, tm_sec=52, tm_wday=2, tm_yday=27, tm_isdst=-1) <= time.struct_time(tm_year=2021, tm_mon=1, tm_mday=27, tm_hour=12, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=27, tm_isdst=0) Ignoring malparsed backtrace: 2021-01-27T12:03:42.890 DEBUG:teuthology.orchestra.run.smithi053:> sync && sudo umount -f /var/lib/ceph/osd/ceph-5 , 2021-01-27T12:03:42.978 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-6 on ubuntu@smithi053.front.sepia.ceph.com , 2021-01-27T12:03:42.978 DEBUG:teuthology.orchestra.run.smithi053:> sync && sudo umount -f /var/lib/ceph/osd/ceph-6 Ignoring malparsed backtrace: 2021-01-27T12:04:00.247 DEBUG:teuthology.orchestra.run.smithi044:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log , 2021-01-27T12:04:00.325 DEBUG:teuthology.orchestra.run.smithi053:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log , 2021-01-27T12:04:00.389 DEBUG:teuthology.orchestra.run.smithi044:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av '\(comm="dmidecode"\|chronyd.service\|name="cephtest"\|scontext=system_u:system_r:nrpe_t:s0\|scontext=system_u:system_r:pcp_pmlogger_t\|scontext=system_u:system_r:pcp_pmcd_t:s0\|comm="rhsmd"\|scontext=system_u:system_r:syslogd_t:s0\|tcontext=system_u:system_r:nrpe_t:s0\|comm="updatedb"\|comm="smartd"\|comm="rhsmcertd-worke"\|comm="setroubleshootd"\|comm="rpm"\|tcontext=system_u:object_r:container_runtime_exec_t:s0\|comm="ksmtuned"\|scontext=system_u:system_r:logrotate_t:s0\)' Ignoring malparsed backtrace: ceph version 17.0.0-184-g507028cb (507028cb839c25d6666833702df6325fd8569096) quincy (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f96d82cb15a] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x5b6) [0x55b230554f36] Ignoring malparsed backtrace: instructed osd(s) 5 to deep-scrub , 2021-01-27T10:56:45.027 DEBUG:teuthology.orchestra.run.smithi122:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2021-01-27T10:56:45.357 INFO:teuthology.orchestra.run.smithi122.stdout: Ignoring malparsed backtrace: 2021-01-27T10:57:47.052 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-0 on ubuntu@smithi122.front.sepia.ceph.com , 2021-01-27T10:57:47.053 DEBUG:teuthology.orchestra.run.smithi122:> sync && sudo umount -f /var/lib/ceph/osd/ceph-0 , 2021-01-27T10:57:47.562 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-1 on ubuntu@smithi122.front.sepia.ceph.com Ignoring malparsed backtrace: 2021-01-27T10:59:39.427 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer , 2021-01-27T10:59:39.440 INFO:teuthology.task.internal:Duration was 866.994880 seconds , 2021-01-27T10:59:39.443 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog Ignoring malparsed backtrace: ceph version 17.0.0-184-g507028cb (507028cb839c25d6666833702df6325fd8569096) quincy (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f96d82cb15a] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x5b6) [0x55b230554f36] Ignoring malparsed backtrace: instructed osd(s) 5 to deep-scrub , 2021-01-27T10:56:45.027 DEBUG:teuthology.orchestra.run.smithi122:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2021-01-27T10:56:45.357 INFO:teuthology.orchestra.run.smithi122.stdout: Ignoring malparsed backtrace: 2021-01-27T10:57:47.052 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-0 on ubuntu@smithi122.front.sepia.ceph.com , 2021-01-27T10:57:47.053 DEBUG:teuthology.orchestra.run.smithi122:> sync && sudo umount -f /var/lib/ceph/osd/ceph-0 , 2021-01-27T10:57:47.562 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-1 on ubuntu@smithi122.front.sepia.ceph.com Ignoring malparsed backtrace: 2021-01-27T10:59:39.427 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer , 2021-01-27T10:59:39.440 INFO:teuthology.task.internal:Duration was 866.994880 seconds , 2021-01-27T10:59:39.443 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog Ignoring malparsed backtrace: ceph version 17.0.0-184-g507028cb (507028cb839c25d6666833702df6325fd8569096) quincy (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f96d82cb15a] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x5b6) [0x55b230554f36] Ignoring malparsed backtrace: instructed osd(s) 5 to deep-scrub , 2021-01-27T10:56:45.027 DEBUG:teuthology.orchestra.run.smithi122:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2021-01-27T10:56:45.357 INFO:teuthology.orchestra.run.smithi122.stdout: Ignoring malparsed backtrace: 2021-01-27T10:57:47.052 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-0 on ubuntu@smithi122.front.sepia.ceph.com , 2021-01-27T10:57:47.053 DEBUG:teuthology.orchestra.run.smithi122:> sync && sudo umount -f /var/lib/ceph/osd/ceph-0 , 2021-01-27T10:57:47.562 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-1 on ubuntu@smithi122.front.sepia.ceph.com Ignoring malparsed backtrace: 2021-01-27T10:59:39.427 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer , 2021-01-27T10:59:39.440 INFO:teuthology.task.internal:Duration was 866.994880 seconds , 2021-01-27T10:59:39.443 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog Ignoring malparsed backtrace: ceph version 17.0.0-184-g507028cb (507028cb839c25d6666833702df6325fd8569096) quincy (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f96d82cb15a] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x5b6) [0x55b230554f36] Ignoring malparsed backtrace: instructed osd(s) 5 to deep-scrub , 2021-01-27T10:56:45.027 DEBUG:teuthology.orchestra.run.smithi122:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2021-01-27T10:56:45.357 INFO:teuthology.orchestra.run.smithi122.stdout: Ignoring malparsed backtrace: 2021-01-27T10:57:47.052 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-0 on ubuntu@smithi122.front.sepia.ceph.com , 2021-01-27T10:57:47.053 DEBUG:teuthology.orchestra.run.smithi122:> sync && sudo umount -f /var/lib/ceph/osd/ceph-0 , 2021-01-27T10:57:47.562 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-1 on ubuntu@smithi122.front.sepia.ceph.com Ignoring malparsed backtrace: 2021-01-27T10:59:39.427 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer , 2021-01-27T10:59:39.440 INFO:teuthology.task.internal:Duration was 866.994880 seconds , 2021-01-27T10:59:39.443 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog Ignoring malparsed backtrace: ceph version 17.0.0-184-g507028cb (507028cb839c25d6666833702df6325fd8569096) quincy (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f96d82cb15a] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x5b6) [0x55b230554f36] Ignoring malparsed backtrace: instructed osd(s) 5 to deep-scrub , 2021-01-27T10:56:45.027 DEBUG:teuthology.orchestra.run.smithi122:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2021-01-27T10:56:45.357 INFO:teuthology.orchestra.run.smithi122.stdout: Ignoring malparsed backtrace: 2021-01-27T10:57:47.052 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-0 on ubuntu@smithi122.front.sepia.ceph.com , 2021-01-27T10:57:47.053 DEBUG:teuthology.orchestra.run.smithi122:> sync && sudo umount -f /var/lib/ceph/osd/ceph-0 , 2021-01-27T10:57:47.562 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-1 on ubuntu@smithi122.front.sepia.ceph.com Ignoring malparsed backtrace: 2021-01-27T10:59:39.427 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer , 2021-01-27T10:59:39.440 INFO:teuthology.task.internal:Duration was 866.994880 seconds , 2021-01-27T10:59:39.443 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog Ignoring malparsed backtrace: ceph version 17.0.0-184-g507028cb (507028cb839c25d6666833702df6325fd8569096) quincy (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f96d82cb15a] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x5b6) [0x55b230554f36] Ignoring malparsed backtrace: instructed osd(s) 5 to deep-scrub , 2021-01-27T10:56:45.027 DEBUG:teuthology.orchestra.run.smithi122:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2021-01-27T10:56:45.357 INFO:teuthology.orchestra.run.smithi122.stdout: Ignoring malparsed backtrace: 2021-01-27T10:57:47.052 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-0 on ubuntu@smithi122.front.sepia.ceph.com , 2021-01-27T10:57:47.053 DEBUG:teuthology.orchestra.run.smithi122:> sync && sudo umount -f /var/lib/ceph/osd/ceph-0 , 2021-01-27T10:57:47.562 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-1 on ubuntu@smithi122.front.sepia.ceph.com Ignoring malparsed backtrace: 2021-01-27T10:59:39.427 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer , 2021-01-27T10:59:39.440 INFO:teuthology.task.internal:Duration was 866.994880 seconds , 2021-01-27T10:59:39.443 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog Ignoring malparsed backtrace: ceph version 17.0.0-184-g507028cb (507028cb839c25d6666833702df6325fd8569096) quincy (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f96d82cb15a] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x5b6) [0x55b230554f36] Ignoring malparsed backtrace: instructed osd(s) 5 to deep-scrub , 2021-01-27T10:56:45.027 DEBUG:teuthology.orchestra.run.smithi122:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2021-01-27T10:56:45.357 INFO:teuthology.orchestra.run.smithi122.stdout: Ignoring malparsed backtrace: 2021-01-27T10:57:47.052 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-0 on ubuntu@smithi122.front.sepia.ceph.com , 2021-01-27T10:57:47.053 DEBUG:teuthology.orchestra.run.smithi122:> sync && sudo umount -f /var/lib/ceph/osd/ceph-0 , 2021-01-27T10:57:47.562 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-1 on ubuntu@smithi122.front.sepia.ceph.com Ignoring malparsed backtrace: 2021-01-27T10:59:39.427 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer , 2021-01-27T10:59:39.440 INFO:teuthology.task.internal:Duration was 866.994880 seconds , 2021-01-27T10:59:39.443 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog Ignoring malparsed backtrace: ceph version 17.0.0-184-g507028cb (507028cb839c25d6666833702df6325fd8569096) quincy (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f96d82cb15a] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x5b6) [0x55b230554f36] Ignoring malparsed backtrace: instructed osd(s) 5 to deep-scrub , 2021-01-27T10:56:45.027 DEBUG:teuthology.orchestra.run.smithi122:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2021-01-27T10:56:45.357 INFO:teuthology.orchestra.run.smithi122.stdout: Ignoring malparsed backtrace: 2021-01-27T10:57:47.052 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-0 on ubuntu@smithi122.front.sepia.ceph.com , 2021-01-27T10:57:47.053 DEBUG:teuthology.orchestra.run.smithi122:> sync && sudo umount -f /var/lib/ceph/osd/ceph-0 , 2021-01-27T10:57:47.562 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-1 on ubuntu@smithi122.front.sepia.ceph.com Ignoring malparsed backtrace: 2021-01-27T10:59:39.427 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer , 2021-01-27T10:59:39.440 INFO:teuthology.task.internal:Duration was 866.994880 seconds , 2021-01-27T10:59:39.443 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog Ignoring malparsed backtrace: ceph version 17.0.0-184-g507028cb (507028cb839c25d6666833702df6325fd8569096) quincy (dev) , 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x7f96d82cb15a] , 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x5b6) [0x55b230554f36] Ignoring malparsed backtrace: instructed osd(s) 5 to deep-scrub , 2021-01-27T10:56:45.027 DEBUG:teuthology.orchestra.run.smithi122:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2021-01-27T10:56:45.357 INFO:teuthology.orchestra.run.smithi122.stdout: Ignoring malparsed backtrace: 2021-01-27T10:57:47.052 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-0 on ubuntu@smithi122.front.sepia.ceph.com , 2021-01-27T10:57:47.053 DEBUG:teuthology.orchestra.run.smithi122:> sync && sudo umount -f /var/lib/ceph/osd/ceph-0 , 2021-01-27T10:57:47.562 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-1 on ubuntu@smithi122.front.sepia.ceph.com Ignoring malparsed backtrace: 2021-01-27T10:59:39.427 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer , 2021-01-27T10:59:39.440 INFO:teuthology.task.internal:Duration was 866.994880 seconds , 2021-01-27T10:59:39.443 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog Found 10 distinct failure reasons Failure: Command failed on smithi139 with status 1: 'sudo TESTDIR=/home/ubuntu/cephtest bash -c \'egrep \'"\'"\'(defer backfill|defer recovery)\'"\'"\' /var/log/ceph/ceph-osd.*.log\'' 1 jobs: ['5832950'] suites: ['mon_election/connectivity', 'msgr-failures/many', 'msgr/async-v1only', 'objectstore/bluestore-bitmap', 'rados', 'rados/singleton/{all/recovery-preemption', 'supported-random-distro$/{ubuntu_latest}}'] Dead: b'2021-01-27T21:05:43.690 DEBUG:teuthology.exit:Finished running handlers' 5 jobs: ['5832990', '5833197', '5833345', '5833025', '5832985'] suites intersection: [] suites union: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{more-active-recovery}', 'backoff/peering_and_degraded', 'ceph', 'clusters/{fixed-2', 'crc-failures/bad_map_crc_failure', 'd-balancer/on', 'fixed-2', 'mode/root', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/osd-delay', 'msgr/async', 'msgr/async-v2only', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp', 'objectstore/bluestore-hybrid', 'openstack', 'openstack}', 'rados', 'rados/cephadm/with-work/{distro/ubuntu_20.04', 'rados/perf/{ceph', 'rados/thrash/{0-size-min-size-overrides/2-size-2-min-size', 'scheduler/dmclock_1Shard_16Threads', 'scheduler/wpq_default_shards', 'settings/optimized', 'start', 'supported-random-distro$/{rhel_8}', 'tasks/rados_api_tests}', 'tasks/rados_python}', 'thrashers/mapgap', 'thrashosds-health', 'ubuntu_latest', 'workloads/cosbench_64K_read_write}', 'workloads/cosbench_64K_write}', 'workloads/rados_api_tests}'] Failure: psutil.NoSuchProcess process no longer exists (pid=4249) 1 jobs: ['5833166'] suites: ['fixed-2', 'mode/root', 'mon_election/connectivity', 'msgr/async-v1only', 'rados/cephadm/with-work/{distro/ubuntu_20.04', 'start', 'tasks/rados_api_tests}'] Failure: "2021-01-27T10:39:36.203354+0000 mon.a (mon.0) 17 : cluster [WRN] Health check failed: 2/9 mons down, quorum a,b,c,d,e,f,g (MON_DOWN)" in cluster log 2 jobs: ['5833018', '5832971'] suites intersection: ['rados'] suites union: ['cluster/{12-osds', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/few', 'msgr-failures/many', 'msgr/async', 'no_pools', 'objectstore/bluestore-stupid', 'objectstore/filestore-xfs', 'openstack}', 'rados', 'rados/multimon/{clusters/9', 'rados/thrash-erasure-code-big/{ceph', 'recovery-overrides/{default}', 'supported-random-distro$/{rhel_8}', 'supported-random-distro$/{ubuntu_latest}', 'tasks/mon_clock_no_skews}', 'thrashers/default', 'thrashosds-health', 'workloads/ec-rados-plugin=jerasure-k=4-m=2}'] Failure: Command failed (workunit test rados/test.sh) on smithi099 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=507028cb839c25d6666833702df6325fd8569096 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rados/test.sh' 2 jobs: ['5833001', '5832972'] suites intersection: ['ceph', 'clusters/{fixed-2', 'msgr/async-v2only', 'openstack}', 'rados'] suites union: ['1-pg-log-overrides/normal_pg_log', '2-recovery-overrides/{more-async-recovery}', 'backoff/peering', 'ceph', 'clusters/{fixed-2', 'crc-failures/bad_map_crc_failure', 'd-balancer/on', 'd-thrash/none', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/few', 'msgr-failures/osd-delay', 'msgr/async-v2only', 'objectstore/bluestore-comp-zstd', 'objectstore/bluestore-stupid', 'openstack}', 'rados', 'rados/thrash/{0-size-min-size-overrides/2-size-2-min-size', 'rados/verify/{centos_latest', 'supported-random-distro$/{centos_8}', 'tasks/rados_api_tests', 'thrashers/mapgap', 'thrashosds-health', 'validater/valgrind}', 'workloads/rados_api_tests}'] Failure: reached maximum tries (50) after waiting for 300 seconds 1 jobs: ['5833325'] suites: ['mon_election/classic', 'objectstore/bluestore-low-osd-mem-target', 'openstack', 'rados/perf/{ceph', 'scheduler/wpq_default_shards', 'settings/optimized', 'ubuntu_latest', 'workloads/radosbench_omap_write}'] Failure: Failed to fetch package version from https://shaman.ceph.com/api/search/?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=nautilus 2 jobs: ['5833231', '5833009'] suites intersection: ['1-install/nautilus', '2-partial-upgrade/firsthalf', '3-thrash/default', '4-workload/{rbd-cls', '5-workload/{radosbench', '6-finish-upgrade', '7-octopus', '8-workload/{rbd-python', 'bluestore-bitmap', 'rados/upgrade/nautilus-x-singleton/{0-cluster/{openstack', 'rbd-import-export', 'rbd_api}', 'readwrite', 'snaps-few-objects}', 'snaps-many-objects}', 'start}', 'thrashosds-health', 'ubuntu_latest}'] suites union: ['1-install/nautilus', '2-partial-upgrade/firsthalf', '3-thrash/default', '4-workload/{rbd-cls', '5-workload/{radosbench', '6-finish-upgrade', '7-octopus', '8-workload/{rbd-python', 'bluestore-bitmap', 'mon_election/classic', 'mon_election/connectivity', 'rados/upgrade/nautilus-x-singleton/{0-cluster/{openstack', 'rbd-import-export', 'rbd_api}', 'readwrite', 'snaps-few-objects}', 'snaps-many-objects}', 'start}', 'thrashosds-health', 'ubuntu_latest}'] Failure: Exiting scrub checking -- not all pgs scrubbed. 1 jobs: ['5832984'] suites: ['1-pg-log-overrides/short_pg_log', '2-recovery-overrides/{more-async-partial-recovery}', 'backoff/peering_and_degraded', 'ceph', 'clusters/{fixed-2', 'crc-failures/default', 'd-balancer/crush-compat', 'mon_election/classic', 'msgr-failures/fastclose', 'msgr/async', 'objectstore/filestore-xfs', 'openstack}', 'rados', 'rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'supported-random-distro$/{centos_8}', 'thrashers/morepggrow', 'thrashosds-health', 'workloads/cache-pool-snaps}'] Failure: "2021-01-27T11:20:41.111224+0000 osd.1 (osd.1) 12 : cluster [ERR] osd.1 found snap mapper error on pg 3.4s0 oid 3:2f1e1119:::smithi13337850-33:e8 snaps missing in mapper, should be: e6,e8 was r -2...repaired" in cluster log 1 jobs: ['5833085'] suites: ['ceph', 'clusters/{fixed-2', 'mon_election/classic', 'msgr-failures/fastclose', 'objectstore/bluestore-hybrid', 'openstack}', 'rados', 'rados/thrash-erasure-code-isa/{arch/x86_64', 'recovery-overrides/{more-async-recovery}', 'supported-random-distro$/{rhel_8}', 'thrashers/default', 'thrashosds-health', 'workloads/ec-rados-plugin=isa-k=2-m=1}'] Failure: Command crashed: 'CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --max-ops 4000 --objects 500 --max-in-flight 16 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 0 --op read 100 --op write 50 --op delete 50 --op copy_from 50 --op write_excl 50 --pool base' 2 jobs: ['5833168', '5833038'] suites intersection: ['mon_election/classic', 'msgr-failures/few', 'objectstore/bluestore-stupid', 'rados'] suites union: ['1-pg-log-overrides/short_pg_log', '2-recovery-overrides/{more-async-partial-recovery}', 'backoff/peering', 'ceph', 'clusters/{fixed-2', 'crc-failures/default', 'd-balancer/crush-compat', 'mon_election/classic', 'msgr-failures/few', 'msgr/async-v1only', 'msgr/async-v2only', 'objectstore/bluestore-stupid', 'openstack}', 'rados', 'rados/singleton/{all/thrash_cache_writeback_proxy_none', 'rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'supported-random-distro$/{centos_8}', 'supported-random-distro$/{rhel_8}}', 'thrashers/default', 'thrashosds-health', 'workloads/cache-agent-small}']